# Render our plots inline
%matplotlib inline

# Import requiered packages
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns

# Supress warnings
import warnings
warnings.filterwarnings('ignore')

url = 'https://data.cityofnewyork.us/api/views/h9gi-nx95/rows.csv?accessType=DOWNLOAD'
df = pd.read_csv(url, low_memory=False)

# Let's take a look at the first 5 rows of the dataframe
df.head(5)

df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2026647 entries, 0 to 2026646
Data columns (total 29 columns):
 #   Column                         Dtype  
---  ------                         -----  
 0   CRASH DATE                     object 
 1   CRASH TIME                     object 
 2   BOROUGH                        object 
 3   ZIP CODE                       object 
 4   LATITUDE                       float64
 5   LONGITUDE                      float64
 6   LOCATION                       object 
 7   ON STREET NAME                 object 
 8   CROSS STREET NAME              object 
 9   OFF STREET NAME                object 
 10  NUMBER OF PERSONS INJURED      float64
 11  NUMBER OF PERSONS KILLED       float64
 12  NUMBER OF PEDESTRIANS INJURED  int64  
 13  NUMBER OF PEDESTRIANS KILLED   int64  
 14  NUMBER OF CYCLIST INJURED      int64  
 15  NUMBER OF CYCLIST KILLED       int64  
 16  NUMBER OF MOTORIST INJURED     int64  
 17  NUMBER OF MOTORIST KILLED      int64  
 18  CONTRIBUTING FACTOR VEHICLE 1  object 
 19  CONTRIBUTING FACTOR VEHICLE 2  object 
 20  CONTRIBUTING FACTOR VEHICLE 3  object 
 21  CONTRIBUTING FACTOR VEHICLE 4  object 
 22  CONTRIBUTING FACTOR VEHICLE 5  object 
 23  COLLISION_ID                   int64  
 24  VEHICLE TYPE CODE 1            object 
 25  VEHICLE TYPE CODE 2            object 
 26  VEHICLE TYPE CODE 3            object 
 27  VEHICLE TYPE CODE 4            object 
 28  VEHICLE TYPE CODE 5            object 
dtypes: float64(4), int64(7), object(18)
memory usage: 448.4+ MB

df.describe()

df.shape

(2026647, 29)

# Number of rows/observations - the data numerosity
df.shape[0]

2026647

# Number of features/column/attributes - the data dimensionality
df.shape[1]

29

df.columns

Index(['CRASH DATE', 'CRASH TIME', 'BOROUGH', 'ZIP CODE', 'LATITUDE',
       'LONGITUDE', 'LOCATION', 'ON STREET NAME', 'CROSS STREET NAME',
       'OFF STREET NAME', 'NUMBER OF PERSONS INJURED',
       'NUMBER OF PERSONS KILLED', 'NUMBER OF PEDESTRIANS INJURED',
       'NUMBER OF PEDESTRIANS KILLED', 'NUMBER OF CYCLIST INJURED',
       'NUMBER OF CYCLIST KILLED', 'NUMBER OF MOTORIST INJURED',
       'NUMBER OF MOTORIST KILLED', 'CONTRIBUTING FACTOR VEHICLE 1',
       'CONTRIBUTING FACTOR VEHICLE 2', 'CONTRIBUTING FACTOR VEHICLE 3',
       'CONTRIBUTING FACTOR VEHICLE 4', 'CONTRIBUTING FACTOR VEHICLE 5',
       'COLLISION_ID', 'VEHICLE TYPE CODE 1', 'VEHICLE TYPE CODE 2',
       'VEHICLE TYPE CODE 3', 'VEHICLE TYPE CODE 4', 'VEHICLE TYPE CODE 5'],
      dtype='object')

df.dtypes

CRASH DATE                        object
CRASH TIME                        object
BOROUGH                           object
ZIP CODE                          object
LATITUDE                         float64
LONGITUDE                        float64
LOCATION                          object
ON STREET NAME                    object
CROSS STREET NAME                 object
OFF STREET NAME                   object
NUMBER OF PERSONS INJURED        float64
NUMBER OF PERSONS KILLED         float64
NUMBER OF PEDESTRIANS INJURED      int64
NUMBER OF PEDESTRIANS KILLED       int64
NUMBER OF CYCLIST INJURED          int64
NUMBER OF CYCLIST KILLED           int64
NUMBER OF MOTORIST INJURED         int64
NUMBER OF MOTORIST KILLED          int64
CONTRIBUTING FACTOR VEHICLE 1     object
CONTRIBUTING FACTOR VEHICLE 2     object
CONTRIBUTING FACTOR VEHICLE 3     object
CONTRIBUTING FACTOR VEHICLE 4     object
CONTRIBUTING FACTOR VEHICLE 5     object
COLLISION_ID                       int64
VEHICLE TYPE CODE 1               object
VEHICLE TYPE CODE 2               object
VEHICLE TYPE CODE 3               object
VEHICLE TYPE CODE 4               object
VEHICLE TYPE CODE 5               object
dtype: object

df["CRASH DATE"] = pd.to_datetime(df["CRASH DATE"], format="%m/%d/%Y")

cols = ["CRASH DATE", "BOROUGH", "NUMBER OF PERSONS INJURED"]
df[cols]
# df[["CRASH DATE", "BOROUGH", "NUMBER OF PERSONS INJURED"]]

df[0:5]

df.loc[0:5, ["CRASH DATE", "BOROUGH", "NUMBER OF PERSONS INJURED"]]

df.iloc[[1,4], 0:3]

# You can also change the value of an observation directly in the data frame
# df.loc[0, "BOROUGH"] = "BROOKLYN"

boolean_condition = (df.LONGITUDE<-50) & (df.LONGITUDE>-74.5) & (df.LATITUDE< 41)
df_filtered = df[boolean_condition]
# same as:
# df_filtered = df[(df.LONGITUDE<-50) & (df.LONGITUDE>-74.5) & (df.LATITUDE< 41)]

# Sum up the number of all injured persons per borough for all different boroughs
df.groupby("BOROUGH", as_index=False)["NUMBER OF PERSONS INJURED"].sum()

# Apply multiple aggregation functions (here "sum" and "max")
df.groupby("BOROUGH", as_index=False)["NUMBER OF PERSONS INJURED"].agg({"SUM INJURED": "sum", "MAX INJURED": "max"})

# Apply an aggregating function on multiple variables by multiple aggregating dimensions
df.groupby(["BOROUGH","VEHICLE TYPE CODE 1"])[["NUMBER OF PERSONS INJURED","CONTRIBUTING FACTOR VEHICLE 1"]].sum()

df["NUMBER OF PERSONS INJURED"].hist()
# df_filtered["NUMBER OF PERSONS INJURED"].plot(kind='hist')

<Axes: >

df["NUMBER OF PERSONS INJURED"].hist(bins=50)

<Axes: >

# A quick exposure to various options of the "hist" command
df["NUMBER OF PERSONS INJURED"].hist(
    bins=20, # use 20 bars
    range=(0,10), # x-axis from 0 to 10
    density=False,  # show normalized count (density=True), or raw counts (density= False)
    figsize=(15,5), # controls the size of the plot
    alpha=0.8, # make the plot 20% transparent
    color='green' # change color
)

<Axes: >

df["NUMBER OF PERSONS INJURED"].plot(
    kind='kde', 
    color='Black', 
    xlim=(0,5), 
    figsize=(15,5)
)

<Axes: ylabel='Density'>

df["BOROUGH"].value_counts()

BOROUGH
BROOKLYN         443172
QUEENS           373975
MANHATTAN        314361
BRONX            206178
STATEN ISLAND     58520
Name: count, dtype: int64

df["BOROUGH"].value_counts().plot(kind='bar')

<Axes: xlabel='BOROUGH'>

# horizontal bars (another way to access column in Pandas when there aren't empty spaces in the column name)
df_filtered.BOROUGH.value_counts().plot(kind='barh')

<Axes: ylabel='BOROUGH'>

pivot = pd.pivot_table(
    data = df,
    index = 'CONTRIBUTING FACTOR VEHICLE 1',
    columns = 'BOROUGH',
    aggfunc = 'count',
    values = 'COLLISION_ID'
)
pivot.head(10)

df["CRASH DATE"].value_counts().head(10)

CRASH DATE
2014-01-21    1161
2018-11-15    1065
2017-12-15     999
2017-05-19     974
2015-01-18     961
2014-02-03     960
2015-03-06     939
2017-05-18     911
2017-01-07     896
2018-03-02     884
Name: count, dtype: int64

df_filtered['CONTRIBUTING FACTOR VEHICLE 1'].value_counts().head(11)

CONTRIBUTING FACTOR VEHICLE 1
Unspecified                       610815
Driver Inattention/Distraction    362502
Failure to Yield Right-of-Way     108874
Following Too Closely              92487
Backing Unsafely                   68667
Other Vehicular                    55679
Passing or Lane Usage Improper     50183
Passing Too Closely                46874
Turning Improperly                 43240
Fatigued/Drowsy                    37696
Unsafe Lane Changing               34075
Name: count, dtype: int64

df_filtered['CONTRIBUTING FACTOR VEHICLE 1'].value_counts()[1:11].plot(kind='barh')

<Axes: ylabel='CONTRIBUTING FACTOR VEHICLE 1'>

plot = (
    df['NUMBER OF PERSONS INJURED'] # take the num of injuries column
    .value_counts() # compute the freuquency of each value
    .sort_index() # sort the results based on the index value instead of the frequency, 
                  # which is the default for value_counts
    .plot( # and plot the results
        kind='line', # we use a line plot because the x-axis is numeric/continuous
        marker='o',  # we use a marker to mark where we have data points 
        logy=True # make the y-axis logarithmic
    )
)
plot.set_xlabel("Number of injuries")
plot.set_ylabel("Number of collisions");

# Date converted to proper date format
df["CRASH DATE"] = pd.to_datetime(df["CRASH DATE"], format="%m/%d/%Y")

(
    df["CRASH DATE"].value_counts() # count the number of accidents per day
    .sort_index() # sort the dates
    .resample('1M') # take periods of 1 month
    .sum() # sum the number of accidents per month
    .drop(pd.to_datetime('2019-04-30'), axis='index') # drop the current month
    .plot() # plot the result
)

<Axes: xlabel='CRASH DATE'>

# We do data filtering by specifying a selection condition to limit the lat/long values 
# to be values idicating the NYC region. Remaining are probably wrong inputs.
cleandf = df[(df.LONGITUDE<-50) & (df.LONGITUDE>-74.5) & (df.LATITUDE< 41)]

cleandf[ (df.LATITUDE>40) & (df.LATITUDE<41) & (df.LONGITUDE> -74.6) & (df.LONGITUDE<-50) ].plot(
    figsize = (20,15),
    kind = 'scatter',
    x = 'LONGITUDE',
    y = 'LATITUDE',
    s = 1, # make each dot to be very small 
    alpha = 0.05 # makes each point 95% transparent
)

<Axes: xlabel='LONGITUDE', ylabel='LATITUDE'>

# Hexbin plot
cleandf.plot(
    kind='hexbin',
    x='LONGITUDE',
    y='LATITUDE',
    gridsize=100,
    cmap=plt.cm.Blues,
    figsize=(10, 7))

<Axes: xlabel='LONGITUDE', ylabel='LATITUDE'>

# Basic 2D density plot
plt.subplots(figsize=(20, 15))

# We take a sample, because density plots take a long time to compute
# and a sample is typically as good as the full dataset
sample = cleandf.sample(10000)

sns.kdeplot(
    x=sample.LONGITUDE,
    y=sample.LATITUDE,
    gridsize=100,  # controls the resolution
    cmap=plt.cm.rainbow,  # color scheme
    shade= True, # whether to have a density plot (True), or just the contours (False)
    alpha=0.5,
    shade_lowest=False,
    n_levels=50  # How many contours/levels to have
)

<Axes: xlabel='LONGITUDE', ylabel='LATITUDE'>

# Basic 2D contour plot
plt.subplots(figsize=(20, 15))

# We take a sample, because density plots take a long time to compute
# and a sample is typically as good as the full dataset
sample = cleandf.sample(10000)

sns.kdeplot(
    x=sample.LONGITUDE,
    y=sample.LATITUDE,
    gridsize=100,
    cmap=plt.cm.rainbow,
    shade=False,
    shade_lowest=False,
    n_levels=25)

<Axes: xlabel='LONGITUDE', ylabel='LATITUDE'>

sample = cleandf.sample(10000)

scatterplot = cleandf.plot(
    kind='scatter',
    x='LONGITUDE',
    y='LATITUDE',
    figsize=(20, 15),
    s=0.5,
    alpha=0.1)

sns.kdeplot(
    x=sample.LONGITUDE,
    y=sample.LATITUDE,
    gridsize=100,
    cmap=plt.cm.rainbow,
    shade=False,
    shade_lowest=False,
    n_levels=20,
    alpha=1,
    ax=scatterplot)

<Axes: xlabel='LONGITUDE', ylabel='LATITUDE'>

# Import
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# Customize plots
%matplotlib inline
sns.set_theme(style="white")
plt.style.use('grayscale')

# Supress warnings
import warnings
warnings.filterwarnings('ignore')

# Create Data
data = {"x1":[0, 0.4, 0.15, 0.05, 0.4, 0.20, 0, 0.45, 1, 0.85, 0.9, 0.7, 0.65, 0.95, 1, 0.8],
"x2":[0.2, 0.35, 0, 0.10, 0.4, 0.25, 0.40, 0.35, 0.85, 0.95, 1, 0.65, 0.75, 0.9, 0.9, 0.95],
"y":[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]}

data = pd.DataFrame(data)
data

# New points
p = pd.DataFrame({"name":["p1", "p2", "p3"], "x1":[0.15, 0.75, 0.5],
"x2":[0.35, 0.8, 0.6]})
p

# Plot
data.plot.scatter("x1", "x2", c="y", colormap="coolwarm_r", figsize=(7, 5))
plt.scatter(p.x1, p.x2, c="orange", marker="x")
for point in p.values:
  plt.text(point[1]+0.01, point[2], point[0])

# Select X and y
X = data[["x1", "x2"]]
y = data["y"]

from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier()
knn.fit(X, y)
y_pred = knn.predict(X)
y_pred

array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1])

knn

KNeighborsClassifier()

KNeighborsClassifier()

# KNN plot
fig, ax = plt.subplots(1, 3, figsize=(18,5))
i = 0
for k in [1, 3, 5]:
  model = KNeighborsClassifier(n_neighbors=k).fit(X,y)
  pred = model.predict(p[["x1", "x2"]])
  ax[i].scatter(data.x1, data.x2, c=data.y, cmap="coolwarm_r")
  ax[i].scatter(p.x1, p.x2, c=pred, cmap="coolwarm_r", marker="x")
  ax[i].set_title("KNN with k = " + str(k))
  i += 1

# Import additional libraries
# data splitting
from sklearn.model_selection import train_test_split
# data scaling
from sklearn.preprocessing import MinMaxScaler
# performance measures
from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import confusion_matrix,f1_score,precision_score,recall_score,accuracy_score,average_precision_score
# classifier
from sklearn.neighbors import KNeighborsClassifier
# set a random seed
np.random.seed(1)

df = pd.read_csv('DiabetesDataset.csv')
print(df.info)
# keep the patient characteristics as inputs x and the diabetes as target y
x = df.drop(columns=['Diabetes'])
y = df['Diabetes'].values
labels = ["No Diabetes", "Diabetes"]

#### START YOUR SOLUTION HERE ####
# Split data into training and test dataset
trainX, testX, trainy, testy = train_test_split(x, y, test_size=0.2)

# Define the data scaler
scaler = MinMaxScaler()

# Fit and transform the training set 
trainX = scaler.fit_transform(trainX)

# Transform the test set
testX = scaler.transform(testX)

# Fit kNN model with k=5 to the training data
model =  KNeighborsClassifier(n_neighbors=5).fit(trainX, trainy)

# # Get predictions on test set
pred = model.predict(testX)

# Compute the performance measures listed in the text above
conf = confusion_matrix(testy, pred)
acc = accuracy_score(testy, pred)
rec = recall_score(testy, pred)
prec = precision_score(testy, pred)
f1 = f1_score(testy, pred)

# Print the values of all performance measures except the confusion matrix 
print( "Performance measurements", "\n",
       "accuracy  : ", round(acc,3),"\n",
       "recall    : ", round(rec,3), "\n",
       "precision : ", round(prec,3),"\n",
       "f1-score  : ", round(f1,3))

# Display confusion matrix using a heatmap
sns.heatmap(conf, 
            annot=True, 
            fmt='d', 
            cbar=False, 
            cmap="coolwarm_r", 
            xticklabels=labels, 
            yticklabels=labels, 
            linewidth = 1)
plt.title("Confusion Matrix")
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.show()
#### END YOUR SOLUTION HERE ####

<bound method DataFrame.info of      Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \
0              6      148             72             35        0  33.6   
1              1       85             66             29        0  26.6   
2              8      183             64              0        0  23.3   
3              1       89             66             23       94  28.1   
4              0      137             40             35      168  43.1   
..           ...      ...            ...            ...      ...   ...   
763           10      101             76             48      180  32.9   
764            2      122             70             27        0  36.8   
765            5      121             72             23      112  26.2   
766            1      126             60              0        0  30.1   
767            1       93             70             31        0  30.4   

     DiabetesPedigreeFunction  Age  Diabetes  
0                       0.627   50         1  
1                       0.351   31         0  
2                       0.672   32         1  
3                       0.167   21         0  
4                       2.288   33         1  
..                        ...  ...       ...  
763                     0.171   63         0  
764                     0.340   27         0  
765                     0.245   30         0  
766                     0.349   47         1  
767                     0.315   23         0  

[768 rows x 9 columns]>
Performance measurements 
 accuracy  :  0.818 
 recall    :  0.673 
 precision :  0.787 
 f1-score  :  0.725

# Predict probabilities for the test set
probs = model.predict_proba(testX)

# Keep the Probabilities of the positive class only
probs = probs[:, 1]

# Function for plotting the ROC curve
def plot_roc_curve(fpr, tpr):
    plt.plot(fpr, tpr, color='orange', label='ROC')
    plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--', label = 'random classifier')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic (ROC) Curve')
    plt.legend()
    plt.show()

# Function for plotting the Precision-Recall curve
def plot_rpc(recall, precision):
    plt.plot(recall, precision, color='orange', label='RPC')
    plt.ylabel('Precision')
    plt.xlabel('Recall = True Positive Rate')
    plt.title('Recall-Precision Curve')
    plt.legend()
    plt.show()


#### START YOUR SOLUTION HERE ####
# Plot ROC curve (check out the function roc_curve)
fpr, tpr, thresholds = roc_curve(testy, probs)
plot_roc_curve(fpr, tpr)

# Compute the Area Under the ROC Curve (AUC) - the ROC AUC score
auc = roc_auc_score(testy, probs)
print("AUC: " , round(auc, 3))

# Plot Precision-Recall curve  
precision, recall, _ = precision_recall_curve(testy, probs)
plot_rpc(recall, precision)
# Compute average precision - Precision-Recall AUC
average_precision = average_precision_score(testy, probs)
print("Average Precision: ", round(average_precision, 3))

#### END YOUR SOLUTION HERE ####

AUC:  0.838

Average Precision:  0.723

# Perform a k-NN on the given dataset and plot the confusion matrix
# compute number of samples
n = len(trainy)

for k in [1, n]:
    #### START YOUR SOLUTION HERE ####
    # Fit a kNN classifier
    knn = KNeighborsClassifier(n_neighbors=k).fit(trainX, trainy)
    # Compute the predictions on the test data using the trained model
    pred = knn.predict(testX)
    # Compute accuracy
    acc_score = accuracy_score(testy, pred)
    # Compute the confusion matrix
    conf = confusion_matrix(testy, pred)
    # Plot the confusion matrix using a heatmap
    sns.heatmap(conf, 
                annot=True, 
                fmt='d', 
                cbar=False, 
                cmap="coolwarm_r", 
                xticklabels=labels, 
                yticklabels=labels, 
                linewidth=1)
    plt.title('Accuracy score for k={} : {}'.format(k, round(acc_score, 3)))
    plt.xlabel('Predicted Label')
    plt.ylabel('True Label')
    plt.show()
    
    #### END YOUR SOLUTION HERE ####

# Import standard libraries
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pylab as plt
%matplotlib inline

# Import to load arff file from url
from scipy.io import arff
import urllib.request
import io

# Sklearn import
from sklearn.model_selection import train_test_split # Splitting the data set
from sklearn.model_selection import KFold, cross_val_score # Cross validation
from sklearn.preprocessing import MinMaxScaler # Normalization
from sklearn.preprocessing import PolynomialFeatures # Polynomial features
from sklearn.preprocessing import LabelEncoder #Label encoding
from sklearn.preprocessing import OneHotEncoder # 1-hot encoding
from sklearn.linear_model import LinearRegression # Regression linear model
from sklearn.linear_model import Lasso # Lasso model
from sklearn.linear_model import Ridge # Ridge model
from sklearn.linear_model import LassoCV # Lasso with cross validation
from sklearn.linear_model import RidgeCV # Ridge with cross validation
from sklearn.linear_model import ElasticNet # ElasticNet model
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score # Metrics for errors

#Load the dataset
weather = pd.read_csv('weather.csv').drop_duplicates().dropna() # drop duplicates and NaN values

# Display a sample of the data
display(weather.head())

#Print the data types
print(weather.dtypes)

Date              object
Location          object
MinTemp          float64
MaxTemp          float64
Rainfall         float64
Evaporation      float64
Sunshine         float64
WindGustDir       object
WindGustSpeed    float64
WindDir9am        object
WindDir3pm        object
WindSpeed9am     float64
WindSpeed3pm       int64
Humidity9am        int64
Humidity3pm        int64
Pressure9am      float64
Pressure3pm      float64
Cloud9am           int64
Cloud3pm           int64
Temp9am          float64
Temp3pm          float64
RainToday         object
RISK_MM          float64
RainTomorrow      object
dtype: object

# Select features of interest
weather3pm = weather.loc[:,['Temp3pm','Humidity3pm', 'Cloud3pm', 'Pressure3pm', 'WindSpeed3pm', 'WindDir3pm', 'Sunshine', 'Rainfall']]

# Summary statistics
display(weather3pm.describe())

# Correlation matrix
display(weather3pm.corr(numeric_only = True))

X = weather3pm[['Humidity3pm']]
y = weather3pm[['Temp3pm']]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0, shuffle=True)

from sklearn.linear_model import LinearRegression

# There are three steps to model something with sklearn
# 1. Set up the model
model = LinearRegression(fit_intercept= True)
# 2. Use fit
model.fit(X_train, y_train)
# 3. Check the score/accuracy
print("R\u00b2 Score of the model: ", round(model.score(X_train, y_train), 3))

R² Score of the model:  0.313

print("Intercept: ", model.intercept_[0])
print("Features coefficients (weigths): ", model.coef_.flatten()[0])

Intercept:  29.543003059769497
Features coefficients (weigths):  -0.22661055784458156

humidity_test = X_test.iloc[0].values[0]
temperature_predicted = model.predict([[humidity_test]]).flatten()[0]
temperature_test = y_test.iloc[0].values[0]
print(f"Prediction/observed temperature for humidity {humidity_test}: {temperature_predicted:.1f}°C vs {temperature_test}°C")

Prediction/observed temperature for humidity 28: 23.2°C vs 27.0°C

/Users/bogo/opt/anaconda3/envs/MachLe310/lib/python3.10/site-packages/sklearn/base.py:464: UserWarning: X does not have valid feature names, but LinearRegression was fitted with feature names
  warnings.warn(

# Model prediction from X_test
predictions = model.predict(X_test)

# Plot the prediction (the line) over the true value (the dots)
plt.scatter(X_test, y_test)
plt.plot(X_test, predictions, 'r')
plt.title("Humidity level against temperature")
plt.xlabel('Humidity level')
plt.ylabel('Temperature °C')
plt.show()

from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

# Compute the MAE, the MSE and the R^2
mae = mean_absolute_error(y_test, predictions)
mse = mean_squared_error(y_test, predictions)
r2 = r2_score(y_test, predictions)

print(f"MAE: {mae:0.2f}")
print(f"MSE: {mse:0.2f}")
print(f"R\u00b2: {r2:0.2f} " )

MAE: 4.27
MSE: 24.44
R²: 0.37

predictions_train = model.predict(X_train)
mae_train = mean_absolute_error(y_train, predictions_train)
mse_train = mean_squared_error(y_train, predictions_train)
r2_train = r2_score(y_train, predictions_train)

print(f"MAE test set: {mae:0.2f}; MAE train set: {mae_train:0.2f};")
print(f"MSE test set: {mse:0.2f}; MSE train set: {mse_train:0.2f};")
print(f"R\u00b2 test set: {r2:0.2f}; R\u00b2 train set: {r2_train:0.2f};" )

MAE test set: 4.27; MAE train set: 4.77;
MSE test set: 24.44; MSE train set: 31.08;
R² test set: 0.37; R² train set: 0.31;

X = weather[['Humidity3pm', 'Cloud3pm', 'Pressure3pm', 'WindSpeed3pm', 'WindDir3pm', 'Sunshine', 'Rainfall']]
y = weather[['Temp3pm']]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0, shuffle=True)

from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

print(X_train[['WindDir3pm']])
# Extract the column of interest
wind_dir_3pm = X_train[['WindDir3pm']].values.ravel()
wind_dir_3pm_test = X_test[['WindDir3pm']].values.ravel()
#Define the encoder
le = LabelEncoder()
#Fit the encoder
le.fit(wind_dir_3pm)
#Transform the train and the test set
X_train = X_train.assign(WindDir3pm=le.transform(wind_dir_3pm))
X_test = X_test.assign(WindDir3pm=le.transform(wind_dir_3pm_test))
print(X_train[['WindDir3pm']])

    WindDir3pm
1            W
333         NW
8          ENE
232        WNW
101          W
..         ...
361         NW
204        SSW
119         SE
47           E
179        WNW

[262 rows x 1 columns]
     WindDir3pm
1            13
333           7
8             1
232          14
101          13
..          ...
361           7
204          11
119           9
47            0
179          14

[262 rows x 1 columns]

from sklearn.preprocessing import MinMaxScaler

#Define the scaler
scaler = MinMaxScaler()
#Fit the scaler
scaler.fit(X_train)
#Transform the training and the test set
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)

## Note that these two steps can be merged into one (only for the training set)
# X_train = scaler.fit_transform(X_train)
# X_test = scaler.transform(X_test)

# 1. Set up the model
model = LinearRegression()
# 2. Use fit
model.fit(X_train, y_train)
# 3. Check the score/accuracy
print("R\u00b2 Score of the model: ", round(model.score(X_train, y_train), 3))
# 4. Print the coefficients of the linear model
print("Intercept: ", model.intercept_[0])
model_coeff = pd.DataFrame(model.coef_.flatten(),
                     index=['Humidity3pm', 'Cloud3pm', 'Pressure3pm', 'WindSpeed3pm', 'WindDir3pm', 'Sunshine', 'Rainfall'],
                     columns=['Coefficients multivariate model'])
model_coeff # Get the coefficients, w

R² Score of the model:  0.621
Intercept:  37.93596617948596

# Predict:
predictions = model.predict(X_test)

# Compute the MAE, the MSE and the R^2 on the test set
mae = mean_absolute_error(y_test, predictions)
mse = mean_squared_error(y_test, predictions)
r2 = r2_score(y_test, predictions)

# Compute the MAE, the MSE and the R^2 on the training set
predictions_train = model.predict(X_train)
mae_train = mean_absolute_error(y_train, predictions_train)
mse_train = mean_squared_error(y_train, predictions_train)
r2_train = r2_score(y_train, predictions_train)

print(f"MAE test set: {mae:0.2f}; MAE training set: {mae_train:0.2f};")
print(f"MSE test set: {mse:0.2f}; MSE training set: {mse_train:0.2f};")
print(f"R\u00b2 test set: {r2:0.2f}; R\u00b2 training set: {r2_train:0.2f};" )

MAE test set: 3.22; MAE training set: 3.39;
MSE test set: 15.86; MSE training set: 17.13;
R² test set: 0.59; R² training set: 0.62;

from sklearn.preprocessing import PolynomialFeatures

# We will use a degree 2
poly = PolynomialFeatures(2)
# Transform our training and test set
X_train_poly = poly.fit_transform(X_train)
X_test_poly = poly.transform(X_test)
# Feature name:
X_poly_features = poly.get_feature_names_out(['Humidity3pm', 'Cloud3pm', 'Pressure3pm', 'WindSpeed3pm', 'WindDir3pm', 'Sunshine', 'Rainfall'])
print(X_poly_features)

['1' 'Humidity3pm' 'Cloud3pm' 'Pressure3pm' 'WindSpeed3pm' 'WindDir3pm'
 'Sunshine' 'Rainfall' 'Humidity3pm^2' 'Humidity3pm Cloud3pm'
 'Humidity3pm Pressure3pm' 'Humidity3pm WindSpeed3pm'
 'Humidity3pm WindDir3pm' 'Humidity3pm Sunshine' 'Humidity3pm Rainfall'
 'Cloud3pm^2' 'Cloud3pm Pressure3pm' 'Cloud3pm WindSpeed3pm'
 'Cloud3pm WindDir3pm' 'Cloud3pm Sunshine' 'Cloud3pm Rainfall'
 'Pressure3pm^2' 'Pressure3pm WindSpeed3pm' 'Pressure3pm WindDir3pm'
 'Pressure3pm Sunshine' 'Pressure3pm Rainfall' 'WindSpeed3pm^2'
 'WindSpeed3pm WindDir3pm' 'WindSpeed3pm Sunshine' 'WindSpeed3pm Rainfall'
 'WindDir3pm^2' 'WindDir3pm Sunshine' 'WindDir3pm Rainfall' 'Sunshine^2'
 'Sunshine Rainfall' 'Rainfall^2']

# Set up the model
model_poly = LinearRegression(fit_intercept=False) # we don't need fit intercept since polynomial features function add a column of ones to the data

# Fit
model_poly.fit(X_train_poly, y_train)

# Check the score/accuracy
print("R\u00b2 Score of the model: ", round(model_poly.score(X_train_poly, y_train), 3))

# Print the coefficients of the linear model
model_coeff = pd.DataFrame(model_poly.coef_.flatten(),
                     index=X_poly_features,
                     columns=['Coefficients polynomial model'])
model_coeff # Get the coefficients, w

R² Score of the model:  0.718

# Predict:
predictions = model_poly.predict(X_test_poly)

# Compute the MAE, the MSE and the R^2 on the test set
mae = mean_absolute_error(y_test, predictions)
mse = mean_squared_error(y_test, predictions)
r2 = r2_score(y_test, predictions)

# Compute the MAE, the MSE and the R^2 on the training set
predictions_train = model_poly.predict(X_train_poly)
mae_train = mean_absolute_error(y_train, predictions_train)
mse_train = mean_squared_error(y_train, predictions_train)
r2_train = r2_score(y_train, predictions_train)

print(f"MAE test set: {mae:0.2f}; MAE training set: {mae_train:0.2f};")
print(f"MSE test set: {mse:0.2f}; MSE training set: {mse_train:0.2f};")
print(f"R\u00b2 test set: {r2:0.2f}; R\u00b2 training set: {r2_train:0.2f};" )

MAE test set: 2.94; MAE training set: 2.90;
MSE test set: 13.84; MSE training set: 12.75;
R² test set: 0.64; R² training set: 0.72;

from sklearn.linear_model import Lasso

# Set up the model
lasso_model = Lasso(alpha=0.2, fit_intercept=False)

# Use fit
lasso_model.fit(X_train_poly, y_train)

# Check the score/accuracy
print("R\u00b2 Score of the model: ", round(lasso_model.score(X_train_poly, y_train), 3))

# Print the coefficients of the linear model
model_coeff = pd.DataFrame(lasso_model.coef_.flatten(),
                     index=X_poly_features,
                     columns=['Coefficients Lasso model'])
model_coeff

R² Score of the model:  0.424

# Predict:
predictions = lasso_model.predict(X_test_poly)

# Compute the MAE, the MSE and the R^2 on the test set
mae_lasso = mean_absolute_error(y_test, predictions)
mse_lasso = mean_squared_error(y_test, predictions)
r2_lasso = r2_score(y_test, predictions)

# Compute the MAE, the MSE and the R^2 on the training set
predictions_train = lasso_model.predict(X_train_poly)
mae_train_lasso = mean_absolute_error(y_train, predictions_train)
mse_train_lasso = mean_squared_error(y_train, predictions_train)
r2_train_lasso = r2_score(y_train, predictions_train)

print(f"MAE test set: {mae_lasso:0.2f}; MAE training set: {mae_train_lasso:0.2f};")
print(f"MSE test set: {mse_lasso:0.2f}; MSE training set: {mse_train_lasso:0.2f};")
print(f"R\u00b2 test set: {r2_lasso:0.2f}; R\u00b2 training set: {r2_train_lasso:0.2f};" )

MAE test set: 4.07; MAE training set: 4.20;
MSE test set: 24.21; MSE training set: 26.06;
R² test set: 0.38; R² training set: 0.42;

from sklearn.linear_model import Ridge

# Set up the model
ridge_model = Ridge(alpha=1.0, fit_intercept=False)

# Use fit
ridge_model.fit(X_train_poly, y_train)

# Check the score/accuracy
print("R\u00b2 Score of the model: ", round(ridge_model.score(X_train_poly, y_train), 3))

# Print the coefficients of the linear model
model_coeff = pd.DataFrame(ridge_model.coef_.flatten(),
                     index=X_poly_features,
                     columns=['Coefficients Ridge model'])
model_coeff['Coefficients Lasso model']=lasso_model.coef_.flatten()
model_coeff['Coefficients polynomial model']=model_poly.coef_.flatten()
model_coeff

R² Score of the model:  0.66

# Predict:
predictions = ridge_model.predict(X_test_poly)

# Compute the MAE, the MSE and the R^2 on the test set
mae_ridge = mean_absolute_error(y_test, predictions)
mse_ridge = mean_squared_error(y_test, predictions)
r2_ridge = r2_score(y_test, predictions)

# Compute the MAE, the MSE and the R^2 on the training set
predictions_train = ridge_model.predict(X_train_poly)
mae_train_ridge = mean_absolute_error(y_train, predictions_train)
mse_train_ridge = mean_squared_error(y_train, predictions_train)
r2_train_ridge = r2_score(y_train, predictions_train)

print(f"MAE test set: {mae_ridge:0.2f}; MAE training set: {mae_train_ridge:0.2f};")
print(f"MSE test set: {mse_ridge:0.2f}; MSE training set: {mse_train_ridge:0.2f};")
print(f"R\u00b2 test set: {r2_ridge:0.2f}; R\u00b2 training set: {r2_train_ridge:0.2f};" )

MAE test set: 3.00; MAE training set: 3.19;
MSE test set: 14.00; MSE training set: 15.38;
R² test set: 0.64; R² training set: 0.66;

model_comparison = pd.DataFrame([mae, mse], index=['MAE', 'MSE'], columns=['Polynomial model'])
model_comparison['LASSO']=[mae_lasso, mse_lasso]
model_comparison['Ridge']=[mae_ridge, mse_ridge]
model_comparison

from sklearn.linear_model import RidgeCV

# Set up the model
ridge_cv_model = RidgeCV(cv=5, fit_intercept=False)

# Use fit
ridge_cv_model.fit(X_train_poly, y_train)

# Check the score/accuracy
print("R\u00b2 Score of the model: ", round(ridge_cv_model.score(X_train_poly, y_train), 3))

# Print the coefficients of the linear model
model_coeff['Coefficients Ridge-CV model']=ridge_cv_model.coef_.flatten()
model_coeff

R² Score of the model:  0.701

# Predict:
predictions = ridge_cv_model.predict(X_test_poly)

# Compute the MAE, the MSE and the R^2 on the test set
mae_ridge_cv = mean_absolute_error(y_test, predictions)
mse_ridge_cv = mean_squared_error(y_test, predictions)
r2_ridge_cv = r2_score(y_test, predictions)

# Compute the MAE, the MSE and the R^2 on the training set
predictions_train = ridge_cv_model.predict(X_train_poly)
mae_train_ridge_cv = mean_absolute_error(y_train, predictions_train)
mse_train_ridge_cv = mean_squared_error(y_train, predictions_train)
r2_train_ridge_cv = r2_score(y_train, predictions_train)

print(f"MAE test set: {mae_ridge:0.2f}; MAE training set: {mae_train_ridge:0.2f};")
print(f"MSE test set: {mse_ridge:0.2f}; MSE training set: {mse_train_ridge:0.2f};")
print(f"R\u00b2 test set: {r2_ridge:0.2f}; R\u00b2 training set: {r2_train_ridge:0.2f};" )

MAE test set: 3.00; MAE training set: 3.19;
MSE test set: 14.00; MSE training set: 15.38;
R² test set: 0.64; R² training set: 0.66;

model_comparison['Ridge with Cross Validation']=[mae_ridge_cv, mse_ridge_cv]
model_comparison

df = pd.read_csv('energy-efficiency.csv', index_col=0)

# YOUR CODE HERE

# Display a sample of the data
display(df.head())

# Display the data types
display(df.dtypes)

# Summary statistics
display(df.describe())

# Correlation matrix
df.corr()

Relative Compactness         float64
Surface Area                 float64
Wall Area                    float64
Roof Area                    float64
Overall Height               float64
Orientation                  float64
Galzing Area                 float64
Glazing Area Distribution    float64
Heating Load                 float64
Cooling Load                 float64
dtype: object

sns.heatmap(df.corr().round(decimals=2), annot=True, cmap="bwr")
plt.show()

df.plot(
    kind='box',
    subplots=True,
    sharey=False,
    figsize=(15, 5)
)

# increase spacing between subplots
plt.subplots_adjust(wspace=1)
plt.show()

sns.pairplot(df)
plt.show()

/Users/bogo/opt/anaconda3/envs/MachLe310/lib/python3.10/site-packages/seaborn/axisgrid.py:118: UserWarning: The figure layout has changed to tight
  self._figure.tight_layout(*args, **kwargs)

# YOUR CODE HERE

# We select our features:
X = df[['Relative Compactness', 'Surface Area', 'Wall Area', 'Roof Area', 'Overall Height', 'Orientation', 'Galzing Area' , 'Glazing Area Distribution']]
y = df[['Heating Load']]

# We split our dataset:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0, shuffle=True)

# YOUR CODE HERE

# Select scaler:
scaler = MinMaxScaler()
# Fit scaler and transform training set
X_train = scaler.fit_transform(X_train)
# Transform test set
X_test = scaler.transform(X_test)

# YOUR CODE HERE

# Create model
model = LinearRegression()
# Train model
model.fit(X_train, y_train)

LinearRegression()

LinearRegression()

# YOUR CODE HERE

# Compute and display R^2
print("R\u00b2 Score of the model: ", round(model.score(X_train, y_train), 3))

# Print intercept
print("Intercept: ", model.intercept_[0])

# Feature names
features = ['Relative Compactness', 'Surface Area', 'Wall Area', 'Roof Area', 'Overall Height', 'Orientation', 'Galzing Area' , 'Glazing Area Distribution']

# Create dataframe with feature names and associated coefficients
model_coeff = pd.DataFrame(model.coef_.flatten(),
                     index=features,
                     columns=['Coefficients multivariate model'])
model_coeff

R² Score of the model:  0.918
Intercept:  28472140757642.45

def errors_model(X_test, y_test, X_train, y_train, model, model_name):
    """This function computes the MAE, MSE, R^2 of a predictive model, both on the test set and on the training set."""
    # Predict on test set
    predictions = model.predict(X_test)
    # Compute the MAE, the MSE and the R^2 on the test set
    mae = mean_absolute_error(y_test, predictions)
    mse = mean_squared_error(y_test, predictions)
    r2 = r2_score(y_test, predictions)
    # Compute the MAE, the MSE and the R^2 on the training set
    predictions_train = model.predict(X_train)
    mae_train = mean_absolute_error(y_train, predictions_train)
    mse_train = mean_squared_error(y_train, predictions_train)
    r2_train = r2_score(y_train, predictions_train)
    # Print results
    print(f"MAE test set: {mae:0.2f}; MAE training set: {mae_train:0.2f};")
    print(f"MSE test set: {mse:0.2f}; MSE training set: {mse_train:0.2f};")
    print(f"R\u00b2 test set: {r2:0.2f}; R\u00b2 training set: {r2_train:0.2f};" )
    # We also create a dataframe with the results MAE and MSE for the test set
    df_errors = pd.DataFrame([mae, mse], index=['MAE', 'MSE'], columns=[model_name])
    return df_errors

errors_multivariate = errors_model(X_test, y_test, X_train, y_train, model, 'Multivariate')

MAE test set: 2.23; MAE training set: 2.02;
MSE test set: 10.04; MSE training set: 8.15;
R² test set: 0.91; R² training set: 0.92;

# YOUR CODE HERE

# Create polynomial model with 2 degrees
poly = PolynomialFeatures(2)

# Fit model and transform train set
X_train_poly = poly.fit_transform(X_train)

# Transform test set
X_test_poly = poly.transform(X_test)

# Get new feature names
X_poly_features = poly.get_feature_names_out(features)
print(X_poly_features)

['1' 'Relative Compactness' 'Surface Area' 'Wall Area' 'Roof Area'
 'Overall Height' 'Orientation' 'Galzing Area' 'Glazing Area Distribution'
 'Relative Compactness^2' 'Relative Compactness Surface Area'
 'Relative Compactness Wall Area' 'Relative Compactness Roof Area'
 'Relative Compactness Overall Height' 'Relative Compactness Orientation'
 'Relative Compactness Galzing Area'
 'Relative Compactness Glazing Area Distribution' 'Surface Area^2'
 'Surface Area Wall Area' 'Surface Area Roof Area'
 'Surface Area Overall Height' 'Surface Area Orientation'
 'Surface Area Galzing Area' 'Surface Area Glazing Area Distribution'
 'Wall Area^2' 'Wall Area Roof Area' 'Wall Area Overall Height'
 'Wall Area Orientation' 'Wall Area Galzing Area'
 'Wall Area Glazing Area Distribution' 'Roof Area^2'
 'Roof Area Overall Height' 'Roof Area Orientation'
 'Roof Area Galzing Area' 'Roof Area Glazing Area Distribution'
 'Overall Height^2' 'Overall Height Orientation'
 'Overall Height Galzing Area' 'Overall Height Glazing Area Distribution'
 'Orientation^2' 'Orientation Galzing Area'
 'Orientation Glazing Area Distribution' 'Galzing Area^2'
 'Galzing Area Glazing Area Distribution' 'Glazing Area Distribution^2']

# YOUR CODE HERE

# Set up the model
model_poly = LinearRegression(fit_intercept=False) # we don't need fit intercept since polynomial features function add a column of ones to the data
# Fit
model_poly.fit(X_train_poly, y_train)

# Check the score/accuracy
print("R\u00b2 Score of the model: ", round(model_poly.score(X_train_poly, y_train), 3))

R² Score of the model:  0.995

# YOUR CODE HERE

errors_polynomial = errors_model(X_test_poly, y_test, X_train_poly, y_train, model_poly, 'Polynomial')

MAE test set: 0.62; MAE training set: 0.52;
MSE test set: 0.68; MSE training set: 0.48;
R² test set: 0.99; R² training set: 1.00;

# YOUR CODE HERE

# Create and Fit model, all in one line!
ridge_model = Ridge(alpha=1.0, fit_intercept=False).fit(X_train_poly, y_train)

# Compute errors using our function:
errors_ridge = errors_model(X_test_poly, y_test, X_train_poly, y_train, ridge_model, 'Ridge')

MAE test set: 1.94; MAE training set: 1.70;
MSE test set: 7.68; MSE training set: 6.06;
R² test set: 0.93; R² training set: 0.94;

# YOUR CODE HERE

# Create and Fit model, all in one line!
lasso_model = Lasso(alpha=1, fit_intercept=False).fit(X_train_poly, y_train)

# Compute errors using our function:
errors_lasso = errors_model(X_test_poly, y_test, X_train_poly, y_train, lasso_model, 'LASSO')

MAE test set: 3.30; MAE training set: 2.89;
MSE test set: 21.67; MSE training set: 16.49;
R² test set: 0.80; R² training set: 0.83;

y_test.values.flatten()

array([15.18, 10.32, 37.26, 16.95, 32.26, 27.9 , 28.18, 28.95, 29.07,
       23.8 ,  6.4 , 42.5 , 11.22, 43.1 , 41.96, 26.33, 10.7 , 28.09,
       14.65, 12.29, 12.46, 32.71, 10.77, 38.57,  6.04, 14.66, 13.  ,
       14.41, 10.75, 39.89, 12.74, 12.74, 41.26, 12.77, 28.56, 35.99,
       13.97, 35.69, 17.14, 10.85, 11.6 , 39.97, 14.42, 25.17, 19.5 ,
       24.4 , 13.69, 12.12, 28.86, 32.75, 32.24, 12.97, 32.31, 14.96,
       36.95, 10.39, 15.55, 26.19, 40.12, 14.53, 14.72, 12.33, 32.23,
       10.34, 12.65, 11.8 , 38.98, 11.21, 25.36, 39.83, 32.82, 10.53,
       23.89, 17.26, 15.16, 24.94, 32.85, 38.65, 32.84, 14.51, 35.69,
       29.62, 11.69, 14.16, 29.92, 25.38, 24.03, 29.63, 12.85, 17.15,
       14.18, 26.45, 12.5 , 24.63, 40.78, 15.09, 29.47, 14.75,  7.1 ,
       35.64, 36.97, 22.93, 19.34, 32.96, 10.39, 29.88, 42.11, 12.92,
       12.57, 15.55, 17.69, 33.09, 32.13, 15.36, 12.19, 15.36, 11.34,
       36.77, 26.45, 38.82, 35.01, 12.16, 12.57, 36.06, 12.28, 40.6 ,
       14.32, 28.17, 12.32, 32.31, 32.21, 24.63, 35.45, 12.03, 36.7 ,
        7.1 , 29.52, 24.58, 33.21, 14.03, 17.37, 19.52, 29.06, 10.71,
       35.56, 16.74, 29.03, 36.91, 12.34, 14.33, 28.15, 11.33, 13.86,
       14.34])

# YOUR CODE HERE

# Create and fit model, with 5 folds:
lasso_cv_model = LassoCV(cv=5, fit_intercept=False).fit(X_train_poly, y_train.values.flatten())

# Erros:
errors_cv = errors_model(X_test_poly, y_test.values.flatten(), X_train_poly, y_train.values.flatten(), lasso_cv_model, 'LASSO with Cross Validation')

MAE test set: 1.97; MAE training set: 1.79;
MSE test set: 8.59; MSE training set: 6.85;
R² test set: 0.92; R² training set: 0.93;

# YOUR CODE HERE

# compile the list of dataframes you want to merge
df_errors_list = [errors_multivariate, errors_polynomial, errors_ridge, errors_lasso, errors_cv]
model_comparison = pd.concat(df_errors_list, axis=1)
model_comparison

model_coeff = pd.DataFrame(model_poly.coef_.flatten(),
                     index=X_poly_features,
                     columns=['Coefficients polynomial model'])
model_coeff

model_coeff['Coefficients RIDGE model'] = ridge_model.coef_.flatten()
model_coeff['Coefficients LASSO model'] = lasso_model.coef_.flatten()
model_coeff['Coefficients LASSO-CV model'] = lasso_cv_model.coef_.flatten()
model_coeff

# load necessary libraries
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix
import numpy as np
np.random.seed(1)
%matplotlib inline

# get data
df = pd.read_csv('DiabetesDataset.csv')
# keep the patient characteristics as inputs X and the diabetes as target y
X = df.drop(columns=['Diabetes'])
y = df['Diabetes'].values

#split dataset into train and test data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=1)

# check out the size of the training and test datasets
print ("Training Set Size:", len(X_train))
print ("Test Set Size:", len(X_test))

Training Set Size: 691
Test Set Size: 77

# Obtain the validation set
#### START YOUR CODE HERE ####
X_train_val, X_val, y_train_val, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=1)

print ("Training Set Size:", len(X_train_val))
print ("Validation Set Size:", len(X_val))
print ("Test Set Size:", len(X_test))
#### END YOUR CODE HERE ####

Training Set Size: 552
Validation Set Size: 139
Test Set Size: 77

# single crossvalidation for given K
from sklearn.neighbors import KNeighborsClassifier

# list to store the scores for the different k values
scores = []
# loop from 1 to 100 to find the best k for k-NN. 
for k in range(1,101):
  #### START YOUR CODE HERE ####
  #train the kNN classifier
  knn_val = KNeighborsClassifier(n_neighbors=k).fit(X_train_val, y_train_val)
  
  # compute the predictions on the validation set
  pred = knn_val.predict(X_val)
  # compute the accuracy score on the validation set 
  val_score = knn_val.score(X_val, y_val)
  # add the accuracy score to your scores list  
  scores.append(val_score)
# find the k that yields the best accuracy score and the best score
best_k = np.argmax(scores)+1
best_score = scores[best_k-1]


# plot the computed accuracy scores for all tested values of k and draw a vertical line at the k
plt.plot(range(1,101), scores)
plt.title("best k at {} with score of {}".format(best_k, round(best_score,3)))
plt.axvline(x=best_k, c="k",  ls="--")
plt.show()
#### END YOUR CODE HERE ####

# Fit the classifier to the training data and evaluate its performance on test set
#### START YOUR SOLUTION HERE ####
knn = KNeighborsClassifier(n_neighbors=best_k).fit(X_train, y_train)
# compute predictions on the test data
pred = knn.predict(X_test)
# compute and print the accuracy on the test data
score = knn.score(X_test, y_test)
print ("Test Accuracy Score: ", score)
# compute the confusion matrix for the test set
conf = confusion_matrix(y_test, pred)
# plot the confusion matrix for the test set using a heatmap
sns.heatmap(conf, 
            annot=True, 
            fmt='d', 
            cbar=False, 
            cmap="coolwarm_r",  
            linewidth=1)
plt.title('Test accuracy score for k={} : {}'.format(best_k, round(score,3)))
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.show()
#### END YOUR SOLUTION HERE ####

Test Accuracy Score:  0.7922077922077922

# single crossvalidation for given K
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score

# arrays to store the mean and standard deviation of the cross validation scores for each tested value of k
mean_scores = np.array([])
scores_std =np.array([])
# loop from 1 to 100 to find the best k for kNN
for k in range(1,101):
#### START YOUR CODE HERE ####
  #create a new kNN model
  knn_cv = KNeighborsClassifier(n_neighbors=k)

  # compute the accuracy scores for each fold of a 10-fold CV
  cv_scores = cross_val_score(knn_cv, X_train, y_train, cv=10)
  
  # compute the average and standard deviation of the CV scores and add them to their respective arrays
  mean_scores = np.append(mean_scores, np.mean(cv_scores))
  scores_std = np.append(scores_std, np.std(cv_scores))

# find the best k and best score
best_k = np.argmax(mean_scores)+1
best_score = mean_scores[best_k-1]
# plot all scores for all tested values of k with their standard deviations and a vertical line the depicts the best k
plt.plot(range(1,101), mean_scores)
plt.title("best k at {} with score of {}".format(best_k, round(best_score,3)))
plt.fill_between(range(0, len(mean_scores)), mean_scores + scores_std, mean_scores - scores_std, alpha=0.15, color='blue')
plt.axvline(x=best_k, c="k",  ls="--")
plt.show()
#### END YOUR CODE HERE ####

#### START YOUR SOLUTION HERE ####
# train the final kNN classifier to the training data
knn = KNeighborsClassifier(n_neighbors=best_k).fit(X_train, y_train)
# compute predictions for the test data
pred = knn.predict(X_test)
# compute and print the accuracy score on the test data
score = knn.score(X_test, y_test)
print ("Test Accuracy Score: ", score)
# compute the confusion matrix for the test set
conf = confusion_matrix(y_test, pred)
# plot the confusion matrix for the test set using a heatmap
sns.heatmap(conf, 
            annot=True, 
            fmt='d', 
            cbar=False, 
            cmap="coolwarm_r",  
            linewidth=1)
plt.title('Test accuracy score for k={} : {}'.format(best_k, round(score,3)))
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.show()
#### END YOUR SOLUTION HERE ####

Test Accuracy Score:  0.7922077922077922

# Grid Search - hyperparameter tuning when we have more than one parameter
from sklearn.model_selection import GridSearchCV
### START YOUR SOLUTION HERE ###
# define the grid of the three parameters to test
grid = {'n_neighbors':np.arange(1,100),
        'p':np.arange(1,3),
        'weights':['uniform','distance']
       }

# train the kNN model
knn = KNeighborsClassifier()
knn_cv = GridSearchCV(knn, grid, cv=10)
knn_cv.fit(X_train, y_train)

# print the best hyperparameters and the corresponding averaged CV accuracy score
print("Hyperparameters:", knn_cv.best_params_)
print("CV Mean Accuracy Score:", round(knn_cv.best_score_, 4))
### END YOUR SOLUTION HERE ###

Hyperparameters: {'n_neighbors': 12, 'p': 1, 'weights': 'uniform'}
CV Mean Accuracy Score: 0.7611

#### START YOUR SOLUTION HERE ####
# fit a kNN classifier with the best parameters selected in the grid search CV
knn = KNeighborsClassifier(n_neighbors=12, p=1, weights='uniform')
knn.fit(X_train, y_train)
# compute the predictions on the test data using the trained model
pred = knn.predict(X_test)
# compute the test accuracy score
acc_score = knn.score(X_test, y_test)
print ("Test Accuracy Score: ", acc_score)

# compute the confusion matrix for the test set
conf = confusion_matrix(y_test, pred)
# plot the confusion matrix for the test set using a heatmap
sns.heatmap(conf, 
            annot=True, 
            fmt='d', 
            cbar=False, 
            cmap="coolwarm_r",  
            linewidth=1)
plt.title('Test Accuracy score for k={} : {}'.format(k, round(acc_score,3)))
plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.show()
#### END YOUR SOLUTION HERE ####

Test Accuracy Score:  0.7532467532467533

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.tree import DecisionTreeClassifier, plot_tree
%matplotlib inline
np.random.seed(1)
plt.figure(figsize=(30,30))

<Figure size 3000x3000 with 0 Axes>

<Figure size 3000x3000 with 0 Axes>

# Load data
cancer = load_breast_cancer()
X = cancer.data
y = cancer.target

from sklearn.metrics import ConfusionMatrixDisplay

# Apply a decisiontree classifier to the data and visualize your decision tree
#### START YOUR CODE HERE ####
# Split the data into training and test set
trainX, testX, trainy, testy = train_test_split(X, y, test_size=0.3)

# fit model 
clf = DecisionTreeClassifier(min_samples_leaf=1 , max_depth = None)
clf = clf.fit(trainX, trainy)

# Plot the fitted tree
plot_tree(clf, filled=True, feature_names=list(cancer.feature_names))
plt.show()

# compute predictions for test set
pred = clf.predict(testX)
# Compute the accuracy score
acc_score = accuracy_score(testy, pred)

# Compute the confusion matrix
conf = confusion_matrix(testy, pred)

# Plot the confusion matrix 
cm_display = ConfusionMatrixDisplay(conf).plot()
#### END YOUR CODE HERE ###

# Grid Search - tuning tree depth
from sklearn.model_selection import GridSearchCV

#### START YOUR SOLUTION HERE ####
# Define grid for the parameter to test - max_depth from 1 to 10
grid = {'max_depth':np.arange(1,11)}

# Define and fit model
tree = DecisionTreeClassifier()
# Grid search CV with 5-fold cross validation
tree_cv = GridSearchCV(tree, grid, cv=5)
tree_cv.fit(trainX, trainy)

# Plot the fitted tree
plot_tree(tree_cv.best_estimator_, filled=True, feature_names=list(cancer.feature_names))
plt.show()

# Print results
print("Hyperparameters (best max_depth):", tree_cv.best_params_)
print("Training CV Accuracy Score:", round(tree_cv.best_score_, 4))
print("Test Accuracy Score:", round(tree_cv.score(testX, testy), 4))
#### END YOUR SOLUTION HERE ####

Hyperparameters (best max_depth): {'max_depth': 4}
Training CV Accuracy Score: 0.9423
Test Accuracy Score: 0.9357

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import ConfusionMatrixDisplay

#### END YOUR SOLUTION HERE ####
# fit model 
clf = RandomForestClassifier(n_estimators=100, oob_score=True)
clf = clf.fit(trainX, trainy)

# compute predictions for the training and test sets
pred_train = clf.predict(trainX)
pred = clf.predict(testX)

# compute the accuracy scores (test, training and OOB)
acc_test = accuracy_score(pred, testy)
acc_train = accuracy_score(pred_train, trainy)
acc_oob = clf.oob_score_

# print the computed scores
print( "Performance measurements", "\n",
       "training accuracy      : ", round(acc_train,3),"\n",
       "test accuracy       : ", round(acc_test,3), "\n",
       "out of bag accuracy : ", round(acc_oob,3),"\n"
       )

# Compute the confusion matrix
conf = confusion_matrix(testy, pred)
# Plot the confusion matrix using a heatmap
cm_display = ConfusionMatrixDisplay(conf).plot()
#### END YOUR SOLUTION HERE ####

Performance measurements 
 training accuracy      :  1.0 
 test accuracy       :  0.953 
 out of bag accuracy :  0.952

# Define the grid for the number of trees
grid = {'n_estimators': np.arange(100,1000,100)}

# Do a grid search to find the optimal number of trees
rf = RandomForestClassifier(random_state = 42)
rf_cv = GridSearchCV(rf, grid, cv=5)
rf_cv.fit(trainX, trainy)

# print the best hyperparameter
print("Best Hyperparameter (number of trees):", rf_cv.best_params_)
# print the training CV accuracy score
print("Training CV Accuracy Score:", rf_cv.best_score_)
# print the test accuracy score
print("Test Accuracy Score:", rf_cv.score(testX, testy))

Best Hyperparameter (number of trees): {'n_estimators': 200}
Training CV Accuracy Score: 0.952246835443038
Test Accuracy Score: 0.9590643274853801

# retrieve the relative importance of each variable and visualize the importance plot
from sklearn.inspection import permutation_importance

#### START YOUR SOLUTION HERE ####
# get the best model from the grid search CV
best_rf_model = rf_cv.best_estimator_
# compute the feature importances using permutation test
perm_importances = permutation_importance(
    best_rf_model, testX, testy, n_repeats=10, random_state=42, n_jobs=2)
# put them in a Series
forest_importances = pd.Series(perm_importances.importances_mean, index=cancer.feature_names)
# sort them (get the indices of the sorted array to be able to apply it on the errors)
sort_index = np.argsort(forest_importances)[::-1]

# plot the importances
fig, ax = plt.subplots()
forest_importances[sort_index].plot.bar(yerr=perm_importances.importances_std[sort_index], ax=ax)
ax.set_title("Feature importances using permutation on full model")
ax.set_ylabel("Mean accuracy decrease")
fig.tight_layout()
plt.show()
#### END YOUR SOLUTION HERE ####

# get the feature importances from the fitted model
importances = best_rf_model.feature_importances_
# get the standard deviations
std = np.std([tree.feature_importances_ for tree in best_rf_model.estimators_], axis=0)
# put them in a Series
forest_importances = pd.Series(importances, index=cancer.feature_names)
# sort the importances (get the indices of the sorted array to be able to apply it on the errors)
sort_index = np.argsort(forest_importances)[::-1]

# plot them
fig, ax = plt.subplots()
forest_importances[sort_index].plot.bar(yerr=std[sort_index], ax=ax)
ax.set_title("Feature importances using MDI")
ax.set_ylabel("Mean decrease in impurity")
fig.tight_layout()

# Import needed libraries
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from sklearn.metrics import ConfusionMatrixDisplay

from sklearn.metrics import confusion_matrix, accuracy_score

# Customize plots
%matplotlib inline
sns.set_theme(style="white")
# Change style (to make the tree plots look nice)
plt.style.use('classic')

# set example equal to 1, 2 and 3 to look into the different examples
example = 1 # 1, 2 or 3 

if example == 1:
  data = {"Degree":["Apprenticeship", "Apprenticeship", "Master", "Bachelor", "Master", "Apprenticeship", "Bachelor", "Bachelor", "Master", "Master"],
  "Sex":[1, 1, 0, 1, 0, 1, 1, 0, 1, 1], 
  "Salary Class":[0, 0, 1, 0, 1, 0, 0, 1, 1, 1]}
  data = pd.DataFrame(data)
elif example == 2:
  data = {"Age":[20, 16, 50, 23, 36, 33, 41, 22, 27, 57],
  "Degree":["Apprenticeship", "School", "Master", "Bachelor", "Bachelor", "Apprenticeship", "Bachelor", "Bachelor", "Master", "Master"],
  "Sex":[0, 1, 0, 1, 0, 1, 0, 1, 0, 1], 
  "Salary Class":[0, 0, 0, 0, 1, 1, 0, 0, 1, 1]}
  data = pd.DataFrame(data)
elif example == 3:
  data = {"Degree":["Apprenticeship", "School", "Master", "Bachelor", "Bachelor", "Apprenticeship", "Bachelor", "Bachelor", "Master", "Master"],
  "Sex":[0, 1, 0, 1, 0, 1, 0, 1, 0, 1], 
  "Salary Class":[1, 1, 0, 1, 1, 0, 0, 0, 1, 1]}
  data = pd.DataFrame(data)
else:
  data = np.nan
  raise ValueError("'example' should be 1, 2 or 3")

data

# Encode the categorical feature into 0-1 encoding using OneHotEncoder
from sklearn.preprocessing import OneHotEncoder
one_hot = OneHotEncoder()
one_hot_degree = one_hot.fit_transform(data[["Degree"]]).toarray()
one_hot_degree = pd.DataFrame(one_hot_degree, columns=one_hot.get_feature_names_out())
one_hot_degree

# Add (concatenate) your one-hot encoded features back in the dataframe 
data_tree = pd.concat([data, one_hot_degree], axis=1)
data_tree

# Select X and y
X = data_tree.drop(["Degree", "Salary Class"], axis=1)
y = data_tree["Salary Class"]

# Classification
from sklearn.tree import DecisionTreeClassifier, plot_tree
tree = DecisionTreeClassifier()
tree.fit(X, y)
tree.score(X, y)

1.0

plt.figure(figsize=(5,5))
plot_tree(tree, filled=True);

# To see the splits
pd.concat([X, y], axis=1).head(2)

tree.get_depth()

2

# Load dataset
df = pd.read_csv("drug200.csv")
df.head()

df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 200 entries, 0 to 199
Data columns (total 6 columns):
 #   Column       Non-Null Count  Dtype  
---  ------       --------------  -----  
 0   Age          200 non-null    int64  
 1   Sex          200 non-null    object 
 2   BP           200 non-null    object 
 3   Cholesterol  200 non-null    object 
 4   Na_to_K      200 non-null    float64
 5   Drug         200 non-null    object 
dtypes: float64(1), int64(1), object(4)
memory usage: 9.5+ KB

df.describe()

# Analysis of the data (univariate)
fig, ax = plt.subplots(3, 2, figsize=(20,15))
i = 0
j = 0
for var in df:
  if df[var].dtypes == "object":
    sns.countplot(x=df[var], ax=ax[i, j])
  else:
    sns.histplot(df[var], ax=ax[i, j])
  i += 1
  if i == 3:
    i = 0
    j += 1
plt.show()

df.Drug.value_counts(normalize=True)

Drug
DrugY    0.455
drugX    0.270
drugA    0.115
drugC    0.080
drugB    0.080
Name: proportion, dtype: float64

plt.figure(figsize=(10,10))
sns.pairplot(df, hue="Drug")

/Users/bogo/opt/anaconda3/envs/MachLe310/lib/python3.10/site-packages/seaborn/axisgrid.py:118: UserWarning: The figure layout has changed to tight
  self._figure.tight_layout(*args, **kwargs)

<seaborn.axisgrid.PairGrid at 0x7fd081cc7280>

<Figure size 800x800 with 0 Axes>

# Cholesterol-Drug
# Counts of the different combinations
df_CH_Drug = df.groupby(["Drug","Cholesterol"]).size().reset_index(name = "Count")

# Barplot of the counts
plt.figure(figsize = (9,5))
sns.barplot(x = "Drug",y="Count", hue = "Cholesterol",data = df_CH_Drug)
plt.title("Cholesterol -- Drug")
plt.show()

# Blood Pressure (BP) - Drug
# Counts of the different combinations
df_BP_Drug = df.groupby(["Drug","BP"]).size().reset_index(name = "Count")

# Barplot of the counts
plt.figure(figsize = (9,5))
sns.barplot(x = "Drug",y="Count", hue = "BP",data = df_BP_Drug)
plt.title("BP -- Drug")
plt.show()

df

from sklearn.preprocessing import OrdinalEncoder
from sklearn.compose import ColumnTransformer

# instantiate encoder
oe=OrdinalEncoder()
# select variables for label encoding
categorical_cols=['Sex', 'BP', 'Cholesterol', 'Drug']
# set up your preprocessor (name, transformer, columns to transform)
# remainder=passthrough means we keep the remaining features
preprocessor = ColumnTransformer([('categorical', oe, categorical_cols)], remainder='passthrough')
# fit pre-processor
encoded_df = pd.DataFrame(preprocessor.fit_transform(df), columns=['Sex', 'BP', 'Cholesterol', 'Drug', 'Age', 'Na_to_K'])

# Create inputs X and target y
X = encoded_df.drop(["Drug"], axis=1)
y = encoded_df.Drug

# Train test split
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.35, random_state=72)
X_train

from sklearn.dummy import DummyClassifier

# instantiate with the "most frequent" parameter
dummy = DummyClassifier(strategy='most_frequent')

# fit it as if we had no X features to train it on
dummy.fit(None, y_train)

#compute test baseline and store it for later
baseline = dummy.score(None, y_test)
baseline

0.42857142857142855

# Fit model
tree = DecisionTreeClassifier()
tree.fit(X_train, y_train)

# Predict
y_pred = tree.predict(X_test)

# Evaluate model
def accuracy_conf_mat(y_test, y_pred):
  print("Accuracy score:", round(accuracy_score(y_test, y_pred), 4))
  conf_mat = confusion_matrix(y_test, y_pred)
  cm_display = ConfusionMatrixDisplay(conf_mat).plot()

accuracy_conf_mat(y_test, y_pred)

Accuracy score: 0.9714

plt.figure(figsize=(12, 12))
plot_tree(tree, filled=True);

tree.get_depth()

4

# Accuracy in training set
tree.score(X_train, y_train)

1.0

# Try out tree depths from 2 to 5
for depth in [2, 3, 4, 5]:
    tree = DecisionTreeClassifier(max_depth=depth).fit(X_train, y_train)
    y_pred = tree.predict(X_test)
    print("Depth: " + str(depth))
    print(round(accuracy_score(y_test, y_pred), 2))
    plt.figure(figsize=(6, 6))
    plot_tree(tree, filled=True)
    plt.show()
    print("\n\n\n\n")

Depth: 2
0.83



Depth: 3
0.81



Depth: 4
0.97



Depth: 5
0.97

# Grid Search - tuning tree depth
from sklearn.model_selection import GridSearchCV

# Define parameter to test - max_depth from 1 to 6
grid = {'max_depth':np.arange(1,7)}

# Define and fit model
tree = DecisionTreeClassifier()
# Grid search CV with 5-fold cross validation
tree_cv = GridSearchCV(tree, grid, cv=5)
tree_cv.fit(X_train, y_train)

# Print results
print("Hyperparameters (best max_depth):", tree_cv.best_params_)
print("Training CV Accuracy Score:", round(tree_cv.best_score_, 4))
print("Test Accuracy Score:", round(tree_cv.score(X_test, y_test), 4))

Hyperparameters (best max_depth): {'max_depth': 4}
Training CV Accuracy Score: 0.9923
Test Accuracy Score: 0.9714

from sklearn.ensemble import RandomForestClassifier

# fit a random forest classfier
rfc = RandomForestClassifier(random_state = 42)
rfc.fit(X_train,y_train)

# Compute the predictions on the test set
y_test = rfc.predict(X_test)

# Evaluate Model
accuracy_conf_mat(y_test, y_pred)

Accuracy score: 1.0

grid = {'n_estimators': np.arange(100,1000,100),
        'criterion': ['gini','entropy']
       }

rf = RandomForestClassifier(random_state=42)
rf_cv = GridSearchCV(rf, grid, cv=5)
rf_cv.fit(X_train,y_train)

print("Hyperparameters:", rf_cv.best_params_)
print("Training CV Accuracy Score:", rf_cv.best_score_)
print("Test Accuracy Score:", rf_cv.score(X_test,y_test))

Hyperparameters: {'criterion': 'gini', 'n_estimators': 100}
Training CV Accuracy Score: 0.9923076923076923
Test Accuracy Score: 1.0

# Import standard libraries
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline

# Import to load arff file from url
from scipy.io import arff
import urllib.request
import io

# Sklearn import
from sklearn.model_selection import train_test_split # Splitting the data set
from sklearn.preprocessing import MinMaxScaler, StandardScaler # Normalization and standard scaler
from sklearn.preprocessing import LabelEncoder, OneHotEncoder # Label and 1-hot encoding
from sklearn.linear_model import LogisticRegression # Logistic regression model
from sklearn.linear_model import LogisticRegressionCV # Logistic regression with cross-validation
from sklearn.metrics import accuracy_score  # Accuracy
from sklearn.metrics import confusion_matrix # Confusion matrix
from sklearn.metrics import precision_score, recall_score, f1_score  # Precision, recall, and f1 score

#Load the dataset
wines = pd.read_csv("wine-quality-red.csv")

# Display a sample of the data
display(wines.head())

# Print columns
print(wines.columns.values)

['fixed acidity' 'volatile acidity' 'citric acid' 'residual sugar'
 'chlorides' 'free sulfur dioxide' 'total sulfur dioxide' 'density' 'pH'
 'sulphates' 'alcohol' 'quality']

wines.describe()

# Define features and target variable
X = wines.drop(columns='quality')
y = wines['quality']

# Count the number of observations (rows) corresponding to each value
y.value_counts()

quality
1    855
0    744
Name: count, dtype: int64

#Split data set into a train and a test data sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0, shuffle=True)

print(f"The training set has {X_train.shape[0]} observations, and the test set has {X_test.shape[0]} observations.")

The training set has 1199 observations, and the test set has 400 observations.

# Define the scaler
scaler = MinMaxScaler()

# Fit the scaler
scaler.fit(X_train) # here the scaler learns the min and max of each attribute from the training set

# Transform the train and the test set
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)

# 1. Set up our model
model = LogisticRegression(penalty='l2', solver='lbfgs', max_iter=1000)

# 2. Fit our model
model.fit(X_train, y_train)

LogisticRegression(max_iter=1000)

LogisticRegression(max_iter=1000)

# Dataframe with the intercept and coefficients (weights) of the logistic model
model_coeff = pd.DataFrame(np.concatenate((model.intercept_, model.coef_.flatten())),
                     index=["Intercept"] + list(X.columns.values),
                     columns=['Coefficients logistic model'])
model_coeff

y_pred = model.predict(X_test)

# Dataframe with probabilities that our first 5 observations belong to each class
model_proba = pd.DataFrame(model.predict_proba(X_test)[0:4],
                     columns=['Probability poor-quality wine', 'Probability good wine'])
model_proba

# Accuracy on the test set
print('Accuracy of Logistic regression classifier on test set: {:.2f}'
     .format(model.score(X_test, y_test)))

# Accuracy on the training set
print('Accuracy of Logistic regression classifier on training set: {:.2f}'
     .format(model.score(X_train, y_train)))

Accuracy of Logistic regression classifier on test set: 0.75
Accuracy of Logistic regression classifier on training set: 0.74

accuracy_test = accuracy_score(y_test, y_pred)

print(f'Accurary of Logistic regression classifier on test set: {accuracy_test :.2f}')

Accurary of Logistic regression classifier on test set: 0.75

y.value_counts().plot.bar(color=['purple', 'blue'], grid=False)
plt.ylabel('Number of observations')
plt.title('Number of observations of each class in the wine dataset');

# Compute the default rate
quality_0 = wines.loc[wines["quality"] == 0].shape[0]
print('# occurrence of class 0: ', quality_0)
quality_1 = wines.loc[wines["quality"] == 1].shape[0]
print('# occurence of class 1: ', quality_1)
defaultrate = max(quality_0, quality_1)/(wines["quality"].shape[0])
print(f'Default rate = {defaultrate:0.4f}')

# occurrence of class 0:  744
# occurence of class 1:  855
Default rate = 0.5347

confusion_matrix(y_test, y_pred)

array([[137,  48],
       [ 52, 163]])

sns.heatmap(confusion_matrix(y_test, y_pred), annot=True, cmap='Blues', fmt='.4g')
plt.xlabel('Predicted label')
plt.ylabel('True labels')
plt.title('Confusion Matrix');

print('The precision for class 1 (good wines) is: {:0.3f}'.format(precision_score(y_test, y_pred)))
print('The recall for class 1 is: {:0.3f}'.format(recall_score(y_test, y_pred)))
print('The F1 score for class 1 is: {:0.3f}'.format(f1_score(y_test, y_pred)))

The precision for class 1 (good wines) is: 0.773
The recall for class 1 is: 0.758
The F1 score for class 1 is: 0.765

# Precision of each class
model_precision = precision_score(y_test, y_pred, average = None)
# Recall of each class
model_recall = recall_score(y_test, y_pred, average = None)
# F1 score of each class
model_f1 = f1_score(y_test, y_pred, average = None)

# Visualize all results in a dataframe:
model_eval = pd.DataFrame([model_precision, model_recall, model_f1],
                    index = ['Precision', 'Recall', 'F1 score'],
                    columns=['Class 0', 'Class 1'])
model_eval

# Load data
forest_fire = pd.read_csv("forestfires.csv")

# YOUR CODE HERE

# Display a sample of the data
display(forest_fire.head())

# Display the data types
display(forest_fire.dtypes)

X          int64
Y          int64
month     object
day       object
FFMC     float64
DMC      float64
DC       float64
ISI      float64
temp     float64
RH         int64
wind     float64
rain     float64
area     float64
class      int64
dtype: object

# Summary statistics
display(forest_fire.describe())

# Correlation matrix visualized in heat map
sns.heatmap(forest_fire.corr(numeric_only = True).round(decimals=1), annot=True, cmap="bwr")
plt.title('Correlation matrix')
plt.show()

forest_fire[['FFMC', 'DMC', 'DC', 'ISI', 'temp', 'RH', 'wind', 'rain']].plot(
    kind='box',
    subplots=True,
    sharey=False,
    figsize=(15, 5),
    title = 'Box plot of weather and fire index features'
)
plt.subplots_adjust(wspace=1)    # increase spacing between subplots
plt.show()

df_ff = forest_fire[['FFMC', 'DMC', 'DC', 'ISI', 'temp', 'RH', 'wind', 'rain', 'class']]
sns.pairplot(df_ff,
             hue = 'class',
             palette = 'deep')
plt.show()

/Users/bogo/opt/anaconda3/envs/MachLe310/lib/python3.10/site-packages/seaborn/axisgrid.py:118: UserWarning: The figure layout has changed to tight
  self._figure.tight_layout(*args, **kwargs)

# YOUR CODE HERE

X_forest = forest_fire[['temp','rain','FFMC', 'wind']]
y_forest = forest_fire[['class']]

# YOUR CODE HERE

X_train_f, X_test_f, y_train_f, y_test_f = train_test_split(X_forest, y_forest,
                                                            test_size=0.2,
                                                            random_state=9,
                                                            shuffle=True)

# YOUR CODE HERE

# Define the scaler
scaler_f = MinMaxScaler()

# Fit and transform training set
X_train_f = scaler_f.fit_transform(X_train_f)

# Transform test set
X_test_f = scaler_f.transform(X_test_f)

# YOUR CODE HERE

# Set up our model
model_f = LogisticRegression(penalty='l2', solver='lbfgs', max_iter=1000)

# Fit our model
model_f.fit(X_train_f, y_train_f.values.flatten())  # We extract values and flatten y_train_f to obtain an array as input instead of data frame

LogisticRegression(max_iter=1000)

LogisticRegression(max_iter=1000)

# YOUR CODE HERE

# Accuracy on the test set
print('Accuracy of Logistic regression classifier on test set: {:.2f}'
     .format(model_f.score(X_test_f, y_test_f)))

# Accuracy on the training set
print('Accuracy of Logistic regression classifier on training set: {:.2f}'
     .format(model_f.score(X_train_f, y_train_f)))

Accuracy of Logistic regression classifier on test set: 0.52
Accuracy of Logistic regression classifier on training set: 0.55

# YOUR CODE HERE

y_forest.value_counts().plot.bar(color=['purple', 'blue'], grid=False)
plt.ylabel('Number of observations')
plt.title('Number of observations of each class in the forest fires dataset');

# YOUR CODE HERE

# Compute the default rate
fire_0 = forest_fire.loc[forest_fire["class"] == 0].shape[0]
print('# occurrence of class 0: ', fire_0)
fire_1 = forest_fire.loc[forest_fire["class"] == 1].shape[0]
print('# occurence of class 1: ', fire_1)
defaultrate_f = max(fire_0, fire_1)/(forest_fire["class"].shape[0])
print(f'Default rate = {defaultrate_f:0.4f}')

# occurrence of class 0:  247
# occurence of class 1:  270
Default rate = 0.5222

# YOUR CODE HERE

# Predict on test set
y_pred_f = model_f.predict(X_test_f)

# Heatmap of confusion matrix
sns.heatmap(confusion_matrix(y_test_f, y_pred_f), annot=True, cmap='Blues', fmt='.4g')
plt.xlabel('Predicted label')
plt.ylabel('True labels')
plt.title('Confusion Matrix');

print('The precision for class 1 is: {:0.3f}'.format(precision_score(y_test_f, y_pred_f)))
print('The recall for class 1 is: {:0.3f}'.format(recall_score(y_test_f, y_pred_f)))
print('The F1 score for class 1 is: {:0.3f}'.format(f1_score(y_test_f, y_pred_f)))

The precision for class 1 is: 0.518
The recall for class 1 is: 0.811
The F1 score for class 1 is: 0.632

# load required packages
import tensorflow as tf
from tensorflow.keras.datasets import fashion_mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, Dropout, MaxPooling2D, Activation, BatchNormalization
import numpy as np
import matplotlib.pyplot as plt
print(tf.__version__) #version should be at least 1.15.x

2.4.1

# load the training and test data
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

# names of class labels (we have ten classes)
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

train_images = train_images.reshape((len(train_images),28,28))
test_images = test_images.reshape((len(test_images),28,28))

# check the shapes of the training and test data 
print("shape for training (x) data : ", train_images.shape) # should be: 60'000 Images each with 28x28 pixels
print("shape for training (y) data : ", train_labels.shape)  # 60'000 Labels with 10 classes
print("shape for test (x) data     : ", test_images.shape)  # 10'000 Images with 28x28 pixels
print("shape for test (y) data     : ", test_labels.shape)  # 10'000 Labels with 10 classes

shape for training (x) data :  (60000, 28, 28)
shape for training (y) data :  (60000,)
shape for test (x) data     :  (10000, 28, 28)
shape for test (y) data     :  (10000,)

# to give you an overview of the data plot first 25 images with corresponding labels
plt.figure(figsize=(10,10))
for i in range(25):
    plt.subplot(5,5,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(train_images[i], cmap=plt.cm.binary)
    plt.xlabel(class_names[train_labels[i]])
plt.show()

# you can also run this model without adding layers, to see how the most simple model performs
model = Sequential()
#model.add(Flatten(input_shape=(28, 28))) #remove this layer if you want to use Covolutional neural networks

# data reshaped for Convolution2D
train_images=train_images.reshape(60000,28,28,1)
test_images=test_images.reshape(10000,28,28,1)

#### DESIGN YOUR NETWORK HERE ####
model.add(Conv2D(filters = 32, kernel_size=(3,3), strides =1, padding='same', input_shape= (28,28,1), activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.3))
model.add(Conv2D(filters = 64, kernel_size=(4,4), strides =1, padding='same',activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.4))
model.add(Conv2D(filters = 128, kernel_size=(5,5), strides =1, padding='same',activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
#### END YOUR NETWORK DESIGN HERE ####

# final output softmax layer for 10 classes (do not modify this layer)
model.add(Dense(10, activation = 'softmax'))

# print a summary of your model
model.summary()

Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_3 (Conv2D)            (None, 28, 28, 32)        320       
_________________________________________________________________
batch_normalization_3 (Batch (None, 28, 28, 32)        128       
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 14, 14, 32)        0         
_________________________________________________________________
dropout_3 (Dropout)          (None, 14, 14, 32)        0         
_________________________________________________________________
conv2d_4 (Conv2D)            (None, 14, 14, 64)        32832     
_________________________________________________________________
batch_normalization_4 (Batch (None, 14, 14, 64)        256       
_________________________________________________________________
max_pooling2d_4 (MaxPooling2 (None, 7, 7, 64)          0         
_________________________________________________________________
dropout_4 (Dropout)          (None, 7, 7, 64)          0         
_________________________________________________________________
conv2d_5 (Conv2D)            (None, 7, 7, 128)         204928    
_________________________________________________________________
batch_normalization_5 (Batch (None, 7, 7, 128)         512       
_________________________________________________________________
max_pooling2d_5 (MaxPooling2 (None, 3, 3, 128)         0         
_________________________________________________________________
dropout_5 (Dropout)          (None, 3, 3, 128)         0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 1152)              0         
_________________________________________________________________
dense_2 (Dense)              (None, 128)               147584    
_________________________________________________________________
dense_3 (Dense)              (None, 10)                1290      
=================================================================
Total params: 387,850
Trainable params: 387,402
Non-trainable params: 448
_________________________________________________________________

# compile model
# you can modify the parameter optimizer with different optimization methods
model.compile(optimizer = 'adam', 
              loss='sparse_categorical_crossentropy', 
              metrics=['accuracy'])

# specify batch size for the optimization (mini-batch)
batch_size = 128
# specify the number of epochs
epochs = 10

# fit model to train data
# we use 10% of the data as validation set
# you can add more 'iterations' by raising the parameter epochs
log = model.fit(train_images, 
                train_labels, 
                batch_size=128,
                epochs=10,
                validation_split=0.1)

# plot accuracy per epoch
plt.plot(log.history['accuracy'], label='Training Accuracy')
plt.plot(log.history['val_accuracy'], label='Validation Accuracy')
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend()
plt.grid()

Epoch 1/10
1875/1875 [==============================] - 9s 4ms/step - loss: 0.7303 - accuracy: 0.7470 - val_loss: 0.3498 - val_accuracy: 0.8718
Epoch 2/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.3548 - accuracy: 0.8673 - val_loss: 0.3284 - val_accuracy: 0.8813
Epoch 3/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.3125 - accuracy: 0.8840 - val_loss: 0.2836 - val_accuracy: 0.8962
Epoch 4/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.2854 - accuracy: 0.8944 - val_loss: 0.2748 - val_accuracy: 0.8997
Epoch 5/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.2677 - accuracy: 0.9016 - val_loss: 0.2490 - val_accuracy: 0.9103
Epoch 6/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.2533 - accuracy: 0.9052 - val_loss: 0.2530 - val_accuracy: 0.9086
Epoch 7/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.2460 - accuracy: 0.9084 - val_loss: 0.2372 - val_accuracy: 0.9139
Epoch 8/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.2356 - accuracy: 0.9122 - val_loss: 0.2283 - val_accuracy: 0.9164
Epoch 9/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.2304 - accuracy: 0.9144 - val_loss: 0.2425 - val_accuracy: 0.9132
Epoch 10/10
1875/1875 [==============================] - 8s 4ms/step - loss: 0.2212 - accuracy: 0.9163 - val_loss: 0.2286 - val_accuracy: 0.9187

# compute the loss and accuracy
test_scores = model.evaluate(test_images, test_labels, verbose=0)
print("Test loss:", test_scores[0])
print("Test accuracy:", test_scores[1])

# you can also make predictions for the test data
predictions = model.predict(test_images)

from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
# get the maximum probability class
max_probability_predictions = np.argmax(predictions, axis=1)
# compute and display the confusion matrix
conf_mat = confusion_matrix(test_labels, max_probability_predictions)
conf_mat = ConfusionMatrixDisplay(conf_mat)

conf_mat.plot()

# This function plots the first n predictions with their true label (in brackets) and image
test_images=test_images.reshape(10000,28,28)

def plot_image(i, predictions_array, true_label, img):
  predictions_array, true_label, img = predictions_array, true_label[i], img[i]
  plt.xticks([])
  plt.yticks([])
  plt.imshow(img, cmap=plt.cm.binary)
  predicted_label = np.argmax(predictions_array)
  if predicted_label == true_label:
    color = 'blue'
  else:
    color = 'red'
  plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

# plot prediction and image
num_rows = 7
num_cols = 5
num_images = num_rows*num_cols
plt.figure(figsize=(2*num_cols, 1*num_rows))
for i in range(num_images):
  plt.subplot(num_rows, num_cols, i+1)
  plot_image(i, predictions[i], test_labels, test_images)
plt.tight_layout()
plt.show()

%matplotlib inline

from IPython.display import set_matplotlib_formats, display
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from cycler import cycler

set_matplotlib_formats('pdf', 'png')
plt.rcParams['figure.dpi'] = 300
plt.rcParams['image.cmap'] = "viridis"
plt.rcParams['image.interpolation'] = "none"
plt.rcParams['savefig.bbox'] = "tight"
plt.rcParams['lines.linewidth'] = 2
plt.rcParams['legend.numpoints'] = 1
plt.rcParams['xtick.labelsize']=6
plt.rcParams['ytick.labelsize']=6

np.set_printoptions(precision=3, suppress=True)

citibike=pd.read_pickle('CitibikeDataSet.pkl')
print("Citibike data:\n{}".format(citibike.head()))

plt.figure(figsize=(12, 5))
xticks = pd.date_range(start=citibike.index.min(), end=citibike.index.max(),
                       freq='D')
plt.xticks(xticks.astype("int"), xticks.strftime("%a-%m-%d"), rotation=90, ha="left")
plt.plot(citibike.index.astype("int"),citibike, linewidth=2)
plt.xlabel("Date")
plt.ylabel("Rentals")
plt.grid(True)

Citibike data:
starttime
2015-08-01 00:00:00     3
2015-08-01 03:00:00     0
2015-08-01 06:00:00     9
2015-08-01 09:00:00    41
2015-08-01 12:00:00    39
Freq: 3H, Name: one, dtype: int64

# extract the target values (number of rentals)
y = citibike.values
# convert to POSIX time by dividing by 10**9
X = citibike.index.astype("int64").values.reshape(-1, 1) // 10**9

# use the first 184 data points for training, the rest for testing
n_train = 184

# function to evaluate and plot a regressor on a given feature set
def eval_on_features(features, target, regressor):
    # split the given features into a training and a test set
    X_train, X_test = features[:n_train], features[n_train:]
    # also split the target array 
    y_train, y_test = target[:n_train], target[n_train:]
    regressor.fit(X_train, y_train)
    print("Test-set R^2: {:.2f}".format(regressor.score(X_test, y_test)))
    y_pred = regressor.predict(X_test)
    y_pred_train = regressor.predict(X_train)
    plt.figure(figsize=(10, 3))

    plt.xticks(range(0, len(X), 8), xticks.strftime("%a %m-%d"), rotation=90,
               ha="left")

    plt.plot(range(n_train), y_train, label="train")
    plt.plot(range(n_train, len(y_test) + n_train), y_test, '-', label="test")
    plt.plot(range(n_train), y_pred_train, '--', label="prediction train")

    plt.plot(range(n_train, len(y_test) + n_train), y_pred, '--',
             label="prediction test")
    plt.legend(loc=(1.01, 0))
    plt.xlabel("Date")
    plt.ylabel("Rentals")

from sklearn.ensemble import RandomForestRegressor
regressor = RandomForestRegressor(n_estimators=100, random_state=0)
eval_on_features(X, y, regressor)

Test-set R^2: -0.04

X_hour = citibike.index.hour.values.reshape(-1, 1)

from sklearn.ensemble import RandomForestRegressor
regressor = RandomForestRegressor(n_estimators=100, random_state=0)
eval_on_features(X_hour, y, regressor)

Test-set R^2: 0.60

from sklearn.ensemble import RandomForestRegressor
regressor = RandomForestRegressor(n_estimators=100, random_state=0)


X_hour_week = np.hstack([citibike.index.dayofweek.values.reshape(-1, 1),
                         citibike.index.hour.values.reshape(-1, 1)])
eval_on_features(X_hour_week, y, regressor)

Test-set R^2: 0.84

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge

eval_on_features(X_hour_week, y, LinearRegression())

Test-set R^2: 0.13

from sklearn.preprocessing import OneHotEncoder
# transform using the OneHotEncoder
enc = OneHotEncoder()
X_hour_week_onehot = enc.fit_transform(X_hour_week).toarray()
eval_on_features(X_hour_week_onehot, y, LinearRegression())

Test-set R^2: 0.61

print(X_hour_week_onehot.shape)

(248, 15)

from sklearn.preprocessing import PolynomialFeatures

poly_transformer = PolynomialFeatures(degree=2, interaction_only=True,
                                      include_bias=False)

X_hour_week_onehot_poly = poly_transformer.fit_transform(X_hour_week_onehot)
eval_on_features(X_hour_week_onehot_poly, y, Ridge())

Test-set R^2: 0.85

print(X_hour_week_onehot_poly.shape)

(248, 120)

%matplotlib inline
#from preamble import *

from IPython.display import set_matplotlib_formats, display
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt


set_matplotlib_formats('pdf', 'png')
plt.rcParams['figure.dpi'] = 300
plt.rcParams['image.cmap'] = "viridis"
plt.rcParams['image.interpolation'] = "none"
plt.rcParams['savefig.bbox'] = "tight"
plt.rcParams['lines.linewidth'] = 2

from sklearn.datasets import load_breast_cancer
from sklearn.feature_selection import SelectPercentile
from sklearn.model_selection import train_test_split

cancer = load_breast_cancer()
print(cancer.target_names)
print(cancer.feature_names)

['malignant' 'benign']
['mean radius' 'mean texture' 'mean perimeter' 'mean area'
 'mean smoothness' 'mean compactness' 'mean concavity'
 'mean concave points' 'mean symmetry' 'mean fractal dimension'
 'radius error' 'texture error' 'perimeter error' 'area error'
 'smoothness error' 'compactness error' 'concavity error'
 'concave points error' 'symmetry error' 'fractal dimension error'
 'worst radius' 'worst texture' 'worst perimeter' 'worst area'
 'worst smoothness' 'worst compactness' 'worst concavity'
 'worst concave points' 'worst symmetry' 'worst fractal dimension']

fig, axes = plt.subplots(15, 2, figsize=(10, 20))
malignant = cancer.data[cancer.target == 0]
benign = cancer.data[cancer.target == 1]
ax = axes.ravel()
for i in range(30):
    _, bins = np.histogram(cancer.data[:, i], bins=50)
    ax[i].hist(malignant[:, i], bins=bins, color='b', alpha=.5)
    ax[i].hist(benign[:, i], bins=bins, color='g', alpha=.5)
    ax[i].set_title(cancer.feature_names[i])
    ax[i].set_yticks(())
    ax[0].set_xlabel("Größe des Merkmals")

X_train, X_test, y_train, y_test = train_test_split(
    cancer.data, cancer.target, random_state=0, test_size=.5)
# use f_classif (the default) and SelectPercentile to select 50% of features
select = SelectPercentile(percentile=50)
select.fit(X_train, y_train)
# transform training set
X_train_selected = select.transform(X_train)

print("X_train.shape: {}".format(X_train.shape))
print("X_train_selected.shape: {}".format(X_train_selected.shape))

X_train.shape: (284, 30)
X_train_selected.shape: (284, 15)

mask = select.get_support()
print(mask)
# visualize the mask. black is True, white is False
plt.matshow(mask.reshape(1, -1), cmap='gray_r')
plt.xlabel("Sample index")
plt.yticks(())

[ True False  True  True False  True  True  True False False  True False
  True  True False False False False False False  True False  True  True
 False  True  True  True False False]

([], <a list of 0 Text yticklabel objects>)

# visualize the mask. black is True, white is False
plt.figure(figsize=(15, 10))
n_features = cancer.data.shape[1]
plt.title("Feature Score")
plt.xticks(np.arange(n_features), cancer.feature_names, rotation=90, fontsize=16)
plt.ylabel("Größe des Merkmals")
plt.bar(range(n_features),select.scores_)

<BarContainer object of 30 artists>

mask.reshape(1, -1)

array([[ True, False,  True,  True, False,  True,  True,  True, False,
        False,  True, False,  True,  True, False, False, False, False,
        False, False,  True, False,  True,  True, False,  True,  True,
         True, False, False]])

from sklearn.linear_model import LogisticRegression

# transform test data
X_test_selected = select.transform(X_test)

lr = LogisticRegression(max_iter=5000)
lr.fit(X_train, y_train)
print("Score with all features: {:.3f}".format(lr.score(X_test, y_test)))
lr.fit(X_train_selected, y_train)
print("Score with only selected features: {:.3f}".format(
    lr.score(X_test_selected, y_test)))

Score with all features: 0.958
Score with only selected features: 0.358

from sklearn.tree import DecisionTreeClassifier

tree = DecisionTreeClassifier(max_depth=4, random_state=0)
tree.fit(X_train, y_train)

print("Accuracy on training set: {:.3f}".format(tree.score(X_train, y_train)))
print("Accuracy on test set: {:.3f}".format(tree.score(X_test, y_test)))

print("Feature importances:\n{}".format(tree.feature_importances_))

def plot_feature_importances_cancer(model):
    plt.figure(figsize=(15,10))
    n_features = cancer.data.shape[1]
    plt.barh(range(n_features), model.feature_importances_, align='center')
    plt.yticks(np.arange(n_features), cancer.feature_names,fontsize=12)
    plt.xticks(fontsize=12)
    
    plt.xlabel("Feature importance",fontsize=16)
    plt.ylabel("Feature",fontsize=16)
    plt.ylim(-1, n_features)

plot_feature_importances_cancer(tree)

Accuracy on training set: 0.982
Accuracy on test set: 0.926
Feature importances:
[0.         0.02355646 0.         0.         0.         0.
 0.02214307 0.         0.         0.02775379 0.00513077 0.
 0.         0.         0.         0.         0.         0.
 0.         0.         0.         0.         0.11199707 0.
 0.         0.         0.         0.80941884 0.         0.        ]

from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier
select = SelectFromModel(
    RandomForestClassifier(n_estimators=100, random_state=42),
    threshold="median")

select.fit(X_train, y_train)
X_train_l1 = select.transform(X_train)
print("X_train.shape: {}".format(X_train.shape))
print("X_train_l1.shape: {}".format(X_train_l1.shape))

X_train.shape: (284, 30)
X_train_l1.shape: (284, 15)

mask = select.get_support()
# visualize the mask. black is True, white is False
plt.matshow(mask.reshape(1, -1), cmap='gray_r')
plt.xlabel("Sample index")
plt.yticks(())

([], <a list of 0 Text yticklabel objects>)

X_test_l1 = select.transform(X_test)
score = LogisticRegression(solver='lbfgs',max_iter=1000).fit(X_train_l1, y_train).score(X_test_l1, y_test)
print("Test score: {:.3f}".format(score))

Test score: 0.954

import json
import numpy as np
import pandas as pd
from sklearn.feature_extraction import text
from sklearn.linear_model import LogisticRegression
import sklearn.model_selection as modsel
import sklearn.preprocessing as preproc

DataPath='D:/Downloads/YelpDataset/'
nightlife_subset = pd.read_pickle('nightlife_subset.pkl')
restaurant_subset = pd.read_pickle('restaurant_subset.pkl')

combined = pd.concat([nightlife_subset, restaurant_subset])

combined['target'] = combined.apply(lambda x: 'Nightlife' in x['categories'],
                                    axis=1)

combined

print(combined.shape)

(20000, 6)

# Split into training and test data sets
training_data, test_data = modsel.train_test_split(combined, 
                                                   train_size=0.7, 
                                                   random_state=123)

training_data.shape

(14000, 6)

test_data.shape

(6000, 6)

# Represent the review text as a bag-of-words 
bow_transform = text.CountVectorizer()
X_tr_bow = bow_transform.fit_transform(training_data['text'])

len(bow_transform.vocabulary_)

29944

X_tr_bow.shape

(14000, 29944)

X_te_bow = bow_transform.transform(test_data['text'])

y_tr = training_data['target']
y_te = test_data['target']

def simple_logistic_classify(X_tr, y_tr, X_test, y_test, description, _C=1.0):
    ## Helper function to train a logistic classifier and score on test data
    m = LogisticRegression(C=_C, solver='newton-cg',max_iter=500).fit(X_tr, y_tr)
    s = m.score(X_test, y_test)
    print ('Test score with', description, 'features:', s)
    return m

m1 = simple_logistic_classify(X_tr_bow, y_tr, X_te_bow, y_te, 'bow')

Test score with bow features: 0.7145

X_tr_l2 = preproc.normalize(X_tr_bow, axis=0)
X_te_l2 = preproc.normalize(X_te_bow, axis=0)

m2 = simple_logistic_classify(X_tr_l2, y_tr, X_te_l2, y_te, 'l2-normalized')

Test score with l2-normalized features: 0.739

# Create the tf-idf representation using the bag-of-words matrix
tfidf_trfm = text.TfidfTransformer(norm=None)
X_tr_tfidf = tfidf_trfm.fit_transform(X_tr_bow)

X_te_tfidf = tfidf_trfm.transform(X_te_bow)

m3 = simple_logistic_classify(X_tr_tfidf, y_tr, X_te_tfidf, y_te, 'tf-idf')

Test score with tf-idf features: 0.6798333333333333

param_grid_ = {'C': [1e-5, 1e-3, 1e-1, 1e0, 1e1, 1e2]}
bow_search = modsel.GridSearchCV(LogisticRegression(solver='lbfgs',max_iter=1000),
                                 cv=5, param_grid=param_grid_, return_train_score=True)
l2_search  = modsel.GridSearchCV(LogisticRegression(solver='lbfgs',max_iter=500),
                                 cv=5, return_train_score=True, param_grid=param_grid_)
tfidf_search = modsel.GridSearchCV(LogisticRegression(solver='lbfgs',max_iter=500),
                                cv=5, return_train_score=True, param_grid=param_grid_)

bow_search.fit(X_tr_bow, y_tr)

C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
C:\Anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)

GridSearchCV(cv=5, error_score=nan,
             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,
                                          fit_intercept=True,
                                          intercept_scaling=1, l1_ratio=None,
                                          max_iter=1000, multi_class='auto',
                                          n_jobs=None, penalty='l2',
                                          random_state=None, solver='lbfgs',
                                          tol=0.0001, verbose=0,
                                          warm_start=False),
             iid='deprecated', n_jobs=None,
             param_grid={'C': [1e-05, 0.001, 0.1, 1.0, 10.0, 100.0]},
             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,
             scoring=None, verbose=0)

bow_search.best_score_

0.7190000000000001

l2_search.fit(X_tr_l2, y_tr)

GridSearchCV(cv=5, error_score=nan,
             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,
                                          fit_intercept=True,
                                          intercept_scaling=1, l1_ratio=None,
                                          max_iter=500, multi_class='auto',
                                          n_jobs=None, penalty='l2',
                                          random_state=None, solver='lbfgs',
                                          tol=0.0001, verbose=0,
                                          warm_start=False),
             iid='deprecated', n_jobs=None,
             param_grid={'C': [1e-05, 0.001, 0.1, 1.0, 10.0, 100.0]},
             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,
             scoring=None, verbose=0)

l2_search.best_score_

0.7276428571428571

tfidf_search.fit(X_tr_tfidf, y_tr)

C:\Users\wurc\.conda\envs\ML\lib\site-packages\sklearn\linear_model\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.
  "of iterations.", ConvergenceWarning)
C:\Users\wurc\.conda\envs\ML\lib\site-packages\sklearn\linear_model\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.
  "of iterations.", ConvergenceWarning)
C:\Users\wurc\.conda\envs\ML\lib\site-packages\sklearn\linear_model\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.
  "of iterations.", ConvergenceWarning)
C:\Users\wurc\.conda\envs\ML\lib\site-packages\sklearn\linear_model\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.
  "of iterations.", ConvergenceWarning)

GridSearchCV(cv=5, error_score='raise-deprecating',
             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,
                                          fit_intercept=True,
                                          intercept_scaling=1, l1_ratio=None,
                                          max_iter=500, multi_class='warn',
                                          n_jobs=None, penalty='l2',
                                          random_state=None, solver='lbfgs',
                                          tol=0.0001, verbose=0,
                                          warm_start=False),
             iid='warn', n_jobs=None,
             param_grid={'C': [1e-05, 0.001, 0.1, 1.0, 10.0, 100.0]},
             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,
             scoring=None, verbose=0)

tfidf_search.best_score_

0.7340714285714286

bow_search.best_params_

{'C': 0.1}

l2_search.best_params_

{'C': 1.0}

tfidf_search.best_params_

{'C': 0.001}

bow_search.cv_results_

{'mean_fit_time': array([0.14379988, 0.21599741, 1.66459932, 4.06620111, 7.41766043,
        7.58538895]),
 'std_fit_time': array([0.01511577, 0.02066551, 0.09023791, 0.2676194 , 0.31160726,
        0.7963182 ]),
 'mean_score_time': array([0.002598  , 0.00240026, 0.00260324, 0.0028017 , 0.00260191,
        0.00320039]),
 'std_score_time': array([0.00049296, 0.00049049, 0.00049121, 0.00074931, 0.00049081,
        0.00147161]),
 'param_C': masked_array(data=[1e-05, 0.001, 0.1, 1.0, 10.0, 100.0],
              mask=[False, False, False, False, False, False],
        fill_value='?',
             dtype=object),
 'params': [{'C': 1e-05},
  {'C': 0.001},
  {'C': 0.1},
  {'C': 1.0},
  {'C': 10.0},
  {'C': 100.0}],
 'split0_test_score': array([0.56535714, 0.70464286, 0.71285714, 0.69785714, 0.67857143,
        0.66785714]),
 'split1_test_score': array([0.55      , 0.71178571, 0.71964286, 0.7075    , 0.68178571,
        0.66821429]),
 'split2_test_score': array([0.57285714, 0.70892857, 0.71642857, 0.70642857, 0.68321429,
        0.66892857]),
 'split3_test_score': array([0.56857143, 0.72035714, 0.725     , 0.69821429, 0.67964286,
        0.67035714]),
 'split4_test_score': array([0.54535714, 0.71392857, 0.72107143, 0.70428571, 0.67857143,
        0.66321429]),
 'mean_test_score': array([0.56042857, 0.71192857, 0.719     , 0.70285714, 0.68035714,
        0.66771429]),
 'std_test_score': array([0.01077933, 0.00523723, 0.00412434, 0.00407206, 0.00184888,
        0.00240747]),
 'rank_test_score': array([6, 2, 1, 3, 4, 5]),
 'split0_train_score': array([0.56491071, 0.73678571, 0.86758929, 0.95883929, 0.99491071,
        1.        ]),
 'split1_train_score': array([0.55928571, 0.73589286, 0.86803571, 0.96035714, 0.995     ,
        1.        ]),
 'split2_train_score': array([0.56482143, 0.73785714, 0.86598214, 0.95964286, 0.99553571,
        1.        ]),
 'split3_train_score': array([0.56848214, 0.73357143, 0.86669643, 0.95982143, 0.995625  ,
        1.        ]),
 'split4_train_score': array([0.56482143, 0.73428571, 0.86625   , 0.95785714, 0.99464286,
        1.        ]),
 'mean_train_score': array([0.56446429, 0.73567857, 0.86691071, 0.95930357, 0.99514286,
        1.        ]),
 'std_train_score': array([0.0029467 , 0.00157467, 0.00078368, 0.0008719 , 0.00037712,
        0.        ])}

import pickle

results_file = open('tfidf_gridcv_results.pkl', 'wb')
pickle.dump(bow_search, results_file, -1)
pickle.dump(tfidf_search, results_file, -1)
pickle.dump(l2_search, results_file, -1)
results_file.close()

pkl_file = open('tfidf_gridcv_results.pkl', 'rb')
bow_search = pickle.load(pkl_file)
tfidf_search = pickle.load(pkl_file)
l2_search = pickle.load(pkl_file)
pkl_file.close()

search_results = pd.DataFrame.from_dict({'bow': bow_search.cv_results_['mean_test_score'],
                               'tfidf': tfidf_search.cv_results_['mean_test_score'],
                               'l2': l2_search.cv_results_['mean_test_score']})
search_results

%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("whitegrid")

ax = sns.boxplot(data=search_results, width=0.4)
ax.set_ylabel('Accuracy', size=14)
ax.tick_params(labelsize=14)
plt.savefig('tfidf_gridcv_results.png')

m1 = simple_logistic_classify(X_tr_bow, y_tr, X_te_bow, y_te, 'bow', 
                              _C=bow_search.best_params_['C'])
m2 = simple_logistic_classify(X_tr_l2, y_tr, X_te_l2, y_te, 'l2-normalized', 
                              _C=l2_search.best_params_['C'])
m3 = simple_logistic_classify(X_tr_tfidf, y_tr, X_te_tfidf, y_te, 'tf-idf', 
                              _C=tfidf_search.best_params_['C'])

Test score with bow features: 0.7293333333333333
Test score with l2-normalized features: 0.739
Test score with tf-idf features: 0.7413333333333333

bow_search.cv_results_['mean_test_score']

array([0.56042857, 0.71192857, 0.719     , 0.70285714, 0.68035714,
       0.66771429])

import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns

from os import listdir
from os.path import join
from scipy.io import wavfile

import IPython.display as ipd
from librosa.feature import melspectrogram
from librosa import power_to_db
from librosa.effects import trim

# plotting utilities
%matplotlib inline
plt.rcParams["figure.figsize"] = (8, 4)
plt.rcParams["figure.titleweight"] = 'bold' 
plt.rcParams["figure.titlesize"] = 'large'
plt.rcParams['figure.dpi'] = 120
#plt.style.use('fivethirtyeight')

rs = 99

files = 'E:/temp/ML_datasets/recordings'
ds_files = listdir(files)

X = []
y = []
for file in ds_files:
    label = int(file.split("_")[0])
    rate, data = wavfile.read(join(files, file))
    X.append(data.astype(np.float16))
    y.append(label)

len(X), len(y)

(3000, 3000)

np.unique(y, return_counts = True)

(array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
 array([300, 300, 300, 300, 300, 300, 300, 300, 300, 300], dtype=int64))

rate = 8000
def show_length_distribution(signals, rate = 8000):
    sampel_times = [len(x)/rate for x in signals]


    f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw={"height_ratios": (.20, .80)})

    # Add a graph in each part
    sns.boxplot(x = sampel_times, ax=ax_box, linewidth = 0.9, color=  '#9af772')
    sns.histplot(x = sampel_times, ax=ax_hist, bins = 'fd', kde = True)

    # Remove x axis name for the boxplot
    ax_box.set(xlabel='')


    title = 'Audio signal lengths'
    x_label = 'duration (seconds)'
    y_label = 'count'

    plt.suptitle(title)
    ax_hist.set_xlabel(x_label)
    ax_hist.set_ylabel(y_label)
    plt.show()
    return sampel_times


lengths = show_length_distribution(X)

np.mean(lengths)

0.4374343333333333

np.std(lengths)

0.14761839633964627

q = 90
np.percentile(lengths, q)

0.604525

tot_outliers = sum(map(lambda x: x > np.percentile(lengths, q), lengths))
print(f'Values outside {q} percentile: {tot_outliers}')

Values outside 90 percentile: 300

Longest_audio = np.argmax([len(x) for x in X])
plt.plot(X[Longest_audio])
plt.title("Longest audio signal");

plt.grid(True)
plt.xlabel('sample')
plt.ylabel('amplitude (arb.)')
ipd.Audio(X[Longest_audio], rate=rate)

Shortest_audio = np.argmin([len(x) for x in X])
plt.plot(X[Shortest_audio])
plt.title("Shortest audio signal");

plt.grid(True)
plt.xlabel('sample')
plt.ylabel('amplitude (arb.)')
ipd.Audio(X[Shortest_audio], rate=rate)

# by default anything below 10 db is considered as silence
def remove_silence(sample, sr= 8000, top_db = 10):
    """This function removes trailing and leading silence periods of audio signals.
    """
    y = np.array(sample, dtype = np.float64)
    # Trim the beginning and ending silence
    yt, _ = trim(y, top_db= top_db)
    return yt

X_tr = [remove_silence(x) for x in X]

show_length_distribution(X_tr);

plt.plot(X_tr[Longest_audio])
plt.title("Longest recording after trimming");

plt.grid(True); plt.xlabel('sample'); plt.ylabel('amplitude (arb.)')
ipd.Audio(X_tr[Longest_audio], rate=rate)

N = int(rate * 0.8) # 0.8 is the upper limit of trimmed audio length
print(N)

6400

X_uniform = []
for x in X_tr:
    if len(x) < N:
        X_uniform.append(np.pad(x, (0, N - len(x)), constant_values = (0, 0)))
    else:
        X_uniform.append(x[:N])

def into_bins(X, bins = 20):
    """This functions creates bins of same width and computes mean and standard deviation on those bins
    """
    X_mean_sd = []
    for x in X:
        x_mean_sd = []
        As = np.array_split(np.array(x), 20)
        for a in As:
            mean = np.round(a.mean(dtype=np.float64), 4)
            sd = np.round(a.std(dtype=np.float64), 4)
            x_mean_sd.extend([mean, sd])

        X_mean_sd.append(x_mean_sd)
    return np.array(X_mean_sd)

from sklearn.ensemble import RandomForestClassifier as RFC
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, classification_report
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score

from sklearn import svm
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

for bins in range(20,101,20):
    X_mean_sd = into_bins(X_uniform, bins)
    X_train, X_test, y_train, y_test = train_test_split(X_mean_sd, y, test_size = 0.20, random_state = rs)
    clf = RFC()
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    acc = accuracy_score(y_test, y_pred)
    p,r,f,s = precision_recall_fscore_support(y_test, y_pred)
    print(f"for {bins} bins, f-macro average:{f.mean()}, accuracy: {acc}")

for 20 bins, f-macro average:0.5575460723648693, accuracy: 0.565
for 40 bins, f-macro average:0.5899310247582592, accuracy: 0.595
for 60 bins, f-macro average:0.5678999146226718, accuracy: 0.5733333333333334
for 80 bins, f-macro average:0.5559281871127035, accuracy: 0.565
for 100 bins, f-macro average:0.5641751372289103, accuracy: 0.5716666666666667

X_time = into_bins(X_uniform, 60)
X_train, X_test, y_train, y_test = train_test_split(X_time, y, test_size = 0.20, random_state = rs)

param_grid = {
    "n_estimators": [100,150,200],
    "criterion": ["gini", "entropy"],
    "min_impurity_decrease": [0.0,0.05,0.1]
}

clf = RFC(random_state = rs, n_jobs = -1 )
grid_search = GridSearchCV(clf, param_grid, scoring = "f1_macro", cv = 5)
grid_search.fit(X_train, y_train)

print("best Parameters for RF model:\n", grid_search.best_params_)
print("best score:", grid_search.best_score_)
print("\n\n Results on test dataset:\n\n")
y_pred = grid_search.predict(X_test)
print(classification_report(y_test, y_pred))

best Parameters for RF model:
 {'criterion': 'gini', 'min_impurity_decrease': 0.0, 'n_estimators': 150}
best score: 0.5922881417565689


 Results on test dataset:


              precision    recall  f1-score   support

           0       0.65      0.73      0.69        62
           1       0.49      0.48      0.49        58
           2       0.50      0.52      0.51        56
           3       0.38      0.24      0.29        62
           4       0.44      0.44      0.44        63
           5       0.47      0.45      0.46        62
           6       0.79      0.86      0.82        56
           7       0.65      0.64      0.64        58
           8       0.70      0.74      0.72        62
           9       0.51      0.59      0.55        61

    accuracy                           0.57       600
   macro avg       0.56      0.57      0.56       600
weighted avg       0.56      0.57      0.56       600

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import confusion_matrix


# Compute confusion matrix
cm = confusion_matrix(y_test, y_pred)

# Plot the confusion matrix using Seaborn
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt="d", cmap="Blues")
plt.ylabel('Actual label')
plt.xlabel('Predicted label')
plt.title('Confusion Matrix')
plt.show()

# Plot the spectrogram of power on log scale

# fig, ax = plt.subplots(figsize = (8,6))

powerSpectrum, freqenciesFound, time, imageAxis = plt.specgram(X[np.random.randint(100)], Fs=rate, scale = "dB")
cbar = plt.gcf().colorbar(imageAxis)
cbar.set_label('db')
plt.grid()
plt.suptitle("Spectrogram of a signal")
plt.xlabel('Time')
plt.ylabel('Frequency')
plt.show()

def ft_mean_std(X, n, f_s = 8000):
    """Computes mean and std of each n x n block of spectrograms of X
       empty bins contains mean values of that column matrices
       
    Parameters:
        X: 2-d sampling array
        n: number of rows or columns to split spectogram
    Returns:
        A 2-d numpy array - feature Matrix with n x 2 x n features as columns
    """
    X_sp = [] #feature matrix
    for x in X:
        sp = power_to_db(melspectrogram(y=x, n_fft= len(x)))
        x_sp = [] #current feature set
        # split the rows
        for v_split in np.array_split(sp, n, axis = 0):
            # split the columns
            for h_split in np.array_split(v_split, n, axis = 1):
                if h_split.size == 0: #happens when number of culumns < n
                    m = np.median(v_split).__round__(4)
                    sd = np.std(v_split).__round__(4)
                else:
                    m = np.mean(h_split).__round__(4)
                    sd = np.std(h_split).__round__(4)
                x_sp.extend([m,sd])
                
        X_sp.append(x_sp)

    return np.array(X_sp)

X_ft = ft_mean_std(X, 10)
len(X_ft)

3000

models = {"rfc": RFC(random_state=rs, n_jobs=-1)}
scores = {}
for n in range(3,20,2):
    X_ft = ft_mean_std(X, n)
    X_train, X_test, y_train, y_test = train_test_split(X_ft, y, test_size = 0.20, random_state = rs)
    score = []
    for model in models:
        clf = models[model]
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        p,r,f,s = precision_recall_fscore_support(y_test, y_pred)
        score.append((model, np.mean(f)))
    scores[n] = score

rf_scores = [x[0][1] for x in scores.values()]
x = scores.keys()

plt.plot(x, rf_scores, 'o-', label = 'RF')
plt.grid(True)
plt.legend(loc = (1,.8))
plt.suptitle("Model evaluation on different n. of bins")
plt.xlabel("n. of bins")
plt.ylabel('mean f-score')
plt.show()

X_ft = ft_mean_std(X, 10)
X_train, X_test, y_train, y_test = train_test_split(X_ft, y, test_size = 0.20, random_state = rs)

param_grid = {
    "n_estimators": [100,150,200],
    "criterion": ["gini", "entropy"],
    "min_impurity_decrease": [0.0,0.05,0.1]
}

clf = RFC(random_state = rs, n_jobs = -1 )
rf_search = GridSearchCV(clf, param_grid, scoring = "f1_macro", cv = 5)
rf_search.fit(X_train, y_train)

print("best Parameters for RF model:\n", rf_search.best_params_)
print("best score:", rf_search.best_score_)
print("\n\n Results on test dataset:\n\n")
y_pred = rf_search.predict(X_test)
print(classification_report(y_test, y_pred))

best Parameters for RF model:
 {'criterion': 'entropy', 'min_impurity_decrease': 0.0, 'n_estimators': 200}
best score: 0.9485476561620784


 Results on test dataset:


              precision    recall  f1-score   support

           0       0.98      1.00      0.99        62
           1       0.95      0.97      0.96        58
           2       0.98      0.96      0.97        56
           3       0.97      0.98      0.98        62
           4       0.95      0.98      0.97        63
           5       0.98      0.95      0.97        62
           6       0.91      0.95      0.93        56
           7       0.98      1.00      0.99        58
           8       1.00      0.92      0.96        62
           9       0.98      0.98      0.98        61

    accuracy                           0.97       600
   macro avg       0.97      0.97      0.97       600
weighted avg       0.97      0.97      0.97       600

rfc = RFC(n_estimators= 200, criterion= 'gini', min_impurity_decrease= 0.0,random_state = rs, n_jobs = -1 )
scores = cross_val_score(rfc, X_ft, y, cv=10, scoring = 'accuracy', n_jobs = -1)
report = f"""Average accuracy of Random Forest model: {np.mean(scores):.2f}
with a standard deviation of {np.std(scores):.2f}
"""
print(report)

Average accuracy of Random Forest model: 0.92
with a standard deviation of 0.04

# Compute confusion matrix
cm = confusion_matrix(y_test, y_pred)

# Plot the confusion matrix using Seaborn
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt="d", cmap="Blues")
plt.ylabel('Actual label')
plt.xlabel('Predicted label')
plt.title('Confusion Matrix')
plt.show()

def split(array,w_bins):
    """Split a matrix into sub-matrices of equal size."""

    # original dimensions
    rows, cols = array.shape
    # size of sub matrices
    sub_rows = rows//w_bins + 1 * rows%w_bins
    sub_cols = cols//w_bins + 1 * cols%w_bins
    # padding to properly fit
    pad_rows = sub_rows*w_bins - rows
    pad_cols = sub_cols*w_bins - cols
    padded_array = np.pad(array, ((0,pad_rows), (0, pad_cols)))
    
    rows, cols = padded_array.shape
    return (padded_array.reshape(rows//sub_rows, sub_rows, -1, sub_cols)
                 .swapaxes(1, 2)
                 .reshape(-1, sub_rows, sub_cols))


def split_ft_mean_std(X, n):
    """ Computes mean and std of each n x n block of spectrograms of X
        bins are padded with zeros to equaly divide in n x n matrices.
        
    Parameters:
        X: 2-d sampling array
        n: number of rows or columns to split spectogram
    Returns:
        A 2-d numpy array - feature Matrix with n x n x 2 features
    """
    f_s = 8000
    X_sp = [] #feature matrix
    for x in X:
        sp = power_to_db(melspectrogram(y=x, n_fft= len(x)))
        blocks = split(sp,n)
        mean = blocks.mean(axis = (-1,-2))
        std = blocks.std(axis = (-1,-2))
        X_sp.append(np.hstack((mean,std)))
    return np.array(X_sp)

# %timeit -n2 -r1 ft_mean_std(X, 10)

%timeit -n2 -r1 split_ft_mean_std(X, 10)

19.4 s ± 0 ns per loop (mean ± std. dev. of 1 run, 2 loops each)

steps = [('scaler', StandardScaler()), ('SVM', svm.SVC())]
pipeline = Pipeline(steps)

parameteres = {'SVM__C':[5,10,20], 'SVM__kernel':["linear", "poly", "rbf"]}

X_ft = split_ft_mean_std(X, 10)
X_train, X_test, y_train, y_test = train_test_split(X_ft, y, test_size = 0.20, random_state = rs)

svm_search = GridSearchCV(pipeline, param_grid=parameteres, cv=5)
svm_search.fit(X_train, y_train)

print("best Parameters for RF model:\n", svm_search.best_params_)
print("best score:", svm_search.best_score_)
print("\n\n Results on test dataset:\n\n")
y_pred = svm_search.predict(X_test)
print(classification_report(y_test, y_pred))

best Parameters for RF model:
 {'SVM__C': 20, 'SVM__kernel': 'rbf'}
best score: 0.8891666666666668


 Results on test dataset:


              precision    recall  f1-score   support

           0       0.86      0.89      0.87        62
           1       0.98      0.86      0.92        58
           2       0.89      0.89      0.89        56
           3       0.88      0.84      0.86        62
           4       0.94      0.97      0.95        63
           5       0.89      0.92      0.90        62
           6       0.83      0.89      0.86        56
           7       0.87      0.95      0.91        58
           8       0.96      0.89      0.92        62
           9       0.89      0.89      0.89        61

    accuracy                           0.90       600
   macro avg       0.90      0.90      0.90       600
weighted avg       0.90      0.90      0.90       600

steps = [('scaler', StandardScaler()), ('SVM', svm.SVC(C= 20, kernel= 'rbf'))]
pipeline = Pipeline(steps)
scores = cross_val_score(pipeline, X_ft, y, cv=10, scoring = 'accuracy', n_jobs = -1)
report = f"""Average accuracy of SVM model: {np.mean(scores):.2f}
with a standard deviation of {np.std(scores):.2f}
"""
print(report)

Average accuracy of SVM model: 0.85
with a standard deviation of 0.05

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm

%matplotlib inline
np.random.seed(42)
y = np.concatenate((np.repeat(-1,100),np.repeat(1,100))) 
X = np.random.rand(200,2) 
X[:,0] += 0.3*y
np.shape(y)
plt.scatter(X[:,0],X[:,1],c=y, cmap=plt.cm.Paired)
plt.show()

C = 0.4
svc = svm.SVC(kernel='linear', C=C).fit(X,y)

svc.predict([(-0.4,1)])

array([-1])

from sklearn import model_selection
C = 0.03
svc = svm.SVC(kernel='linear', C=C)
loo = model_selection.LeaveOneOut()
# svc.fit(...).score() gives 1 if prediction is correct 0 otherwise 
res = [svc.fit(X[train], y[train]).score(X[test], y[test]) for train, test in loo.split(X)]
#res is a vector with 0,1
np.mean(res) #The average accuracy

0.78

from sklearn import model_selection
Clist = np.logspace(-3,3,14)
for C in Clist:
    svc = svm.SVC(kernel='linear', C=C)
    loo = model_selection.LeaveOneOut()
    # svc.fit(...).score() gives 1 if prediction is correct 0 otherwise 
    res = [svc.fit(X[train], y[train]).score(X[test], y[test]) for train, test in loo.split(X)]
    #res is a vector with 0,1
    print('C: %f \t accuracy: %f' % (C,np.mean(res))) #The average accuracy

C: 0.001000 	 accuracy: 0.000000
C: 0.002894 	 accuracy: 0.000000
C: 0.008377 	 accuracy: 0.000000
C: 0.024245 	 accuracy: 0.780000
C: 0.070170 	 accuracy: 0.780000
C: 0.203092 	 accuracy: 0.780000
C: 0.587802 	 accuracy: 0.765000
C: 1.701254 	 accuracy: 0.790000
C: 4.923883 	 accuracy: 0.785000
C: 14.251027 	 accuracy: 0.785000
C: 41.246264 	 accuracy: 0.785000
C: 119.377664 	 accuracy: 0.785000
C: 345.510729 	 accuracy: 0.785000
C: 1000.000000 	 accuracy: 0.785000

from __future__ import print_function

from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.svm import SVC

n_samples = len(y)

# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.5, random_state=0)

# Set the parameters by cross-validation
tuned_parameters = [{'kernel': ['linear'], 'C': [1, 10, 15, 100, 1000]},
                    {'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 10, 100, 1000]}]

score = 'accuracy'

print("# Tuning hyper-parameters for %s" % score)
print()

clf = GridSearchCV(SVC(C=1), tuned_parameters, cv=5,
                   scoring=score)
clf.fit(X_train, y_train)

print("Best parameters set found on development set:")
print()
print(clf.best_params_)
print()
print("Grid scores on development set:")
print()
results = clf.cv_results_
for i in range(len(results["params"])):
    print("%0.3f (+/-%0.03f) for %r" % (results["mean_test_score"][i], results["std_test_score"][i] * 2, results["params"][i]))

# Tuning hyper-parameters for accuracy

Best parameters set found on development set:

{'C': 1, 'kernel': 'linear'}

Grid scores on development set:

0.790 (+/-0.194) for {'C': 1, 'kernel': 'linear'}
0.780 (+/-0.174) for {'C': 10, 'kernel': 'linear'}
0.780 (+/-0.174) for {'C': 15, 'kernel': 'linear'}
0.790 (+/-0.194) for {'C': 100, 'kernel': 'linear'}
0.790 (+/-0.194) for {'C': 1000, 'kernel': 'linear'}
0.530 (+/-0.049) for {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'}
0.530 (+/-0.049) for {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}
0.530 (+/-0.049) for {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}
0.530 (+/-0.049) for {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}
0.780 (+/-0.136) for {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'}
0.530 (+/-0.049) for {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'}
0.790 (+/-0.194) for {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'}
0.780 (+/-0.136) for {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}

y_true, y_pred = y_test, clf.predict(X_test)
print(classification_report(y_true, y_pred))
np.mean(y_true == y_pred)

              precision    recall  f1-score   support

          -1       0.74      0.85      0.79        47
           1       0.85      0.74      0.79        53

    accuracy                           0.79       100
   macro avg       0.79      0.79      0.79       100
weighted avg       0.80      0.79      0.79       100

0.79

# Plot the decision boundaries and margins of the classifier
def PlotDecisionBoundary(model, X,y):

    #plot decision boundary for model in case of 2D feature space
    x1=X[:,0]
    x2=X[:,1]
    
    # Create grid to evaluate model
    xx = np.linspace(min(x1), max(x1), len(x1))
    yy = np.linspace(min(x1), max(x2), len(x2))
    YY, XX = np.meshgrid(yy, xx)
    xy = np.vstack([XX.ravel(), YY.ravel()]).T

    train_size = len(x1)

    # Assigning different colors to the classes
    colors = y
    colors = np.where(colors == 1, '#8C7298', '#4786D1')

    # Get the separating hyperplane
    Z = model.decision_function(xy).reshape(XX.shape)
    plt.scatter(x1, x2, c=colors)

    # Draw the decision boundary and margins
    plt.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5, linestyles=['--', '-', '--'])

    # Highlight support vectors with a circle around them
    plt.scatter(model.support_vectors_[:, 0], model.support_vectors_[:, 1], s=100,
               linewidth=1, facecolors='none', edgecolors='k')
    plt.title('g=%f | C=%f' % (model.gamma, model.C))

CList=np.logspace(-2,1,4)
gammaList=np.logspace(-2,1,4)
k=0

plt.figure(figsize=(20,20))

for C in CList:
    for gamma in gammaList:
        k=k+1
        svc = svm.SVC(kernel='rbf', C=C, gamma=gamma, probability=True).fit(X,y)
        plt.subplot(len(CList), len(gammaList), k)

        PlotDecisionBoundary(svc, X,y)

%matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns

pima = pd.read_csv('./pima-indians-diabetes.csv', header=None)
pima.columns = ["NumTimesPrg", "PlGlcConc", "BloodP", "SkinThick", "TwoHourSerIns", "BMI", "DiPedFunc", "Age", "HasDiabetes"]

pima.head(5)

perc_diab = pima['HasDiabetes'].mean()

print('percentage of diabetes: %f ' % np.round(perc_diab*100, 2))

y = pima['HasDiabetes']
X = pima.drop('HasDiabetes', axis=1)

X_names = X.columns

percentage of diabetes: 34.900000

pima.describe()

print(X_names.values)

['NumTimesPrg' 'PlGlcConc' 'BloodP' 'SkinThick' 'TwoHourSerIns' 'BMI'
 'DiPedFunc' 'Age']

pima2 = pima.loc[:,X_names.values]
target= pima['HasDiabetes']
pima2.replace(0, np.nan,inplace=True)
pima2['HasDiabetes']=target

pima2.dropna(axis=0,inplace=True)
pima=pima2

pima.describe()

plt.figure()
pima.hist(figsize=(12,12),bins=37)
plt.show()

<Figure size 640x480 with 0 Axes>

corrmatrix=pima.corr()
import seaborn as sns
corrmatrix
sns.heatmap(corrmatrix)

<AxesSubplot:>

from sklearn.model_selection import train_test_split

Xtr, Xtest, ytrain, ytest = train_test_split(pima.loc[:,X_names.values],pima.HasDiabetes,train_size=0.8)

Xtr.head()

from sklearn.preprocessing import StandardScaler, PowerTransformer

scaler = PowerTransformer()

Xtrain_scale = scaler.fit_transform(Xtr)
Xtest_scale = scaler.transform(Xtest)

Xtrain = pd.DataFrame(Xtrain_scale, columns=X.columns)
Xtrain.hist(bins=50, figsize=(20, 15))
plt.show()

from sklearn.svm import SVC
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score

model=SVC(kernel='rbf', gamma=1, C=1)
model.fit(Xtrain, ytrain)
ypred=model.predict(Xtest_scale)
print('Accuracy: %f' % accuracy_score(ytest, ypred))

Accuracy: 0.691176

C:\Users\christoph.wuersch\.conda\envs\ML\lib\site-packages\sklearn\base.py:450: UserWarning: X does not have valid feature names, but SVC was fitted with feature names
  warnings.warn(

C_range = np.logspace(-3, 3, 7)
gamma_range = np.logspace(-3, 3, 7)
param_grid = dict(gamma=gamma_range, C=C_range)

model=SVC()

cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
grid = GridSearchCV(model, param_grid=param_grid, cv=cv,n_jobs=-1)
grid.fit(Xtrain, ytrain)

print("The best parameters are %s with a score of %0.2f"
      % (grid.best_params_, grid.best_score_))

The best parameters are {'C': 100.0, 'gamma': 0.01} with a score of 0.81

from sklearn.metrics import classification_report, confusion_matrix

ypred = grid.best_estimator_.predict(Xtest_scale)
target_names = ['negative', 'positive']
print(classification_report(ytest, ypred, target_names=target_names))
print(confusion_matrix(ytest, ypred))

              precision    recall  f1-score   support

    negative       0.73      0.84      0.78        43
    positive       0.63      0.48      0.55        25

    accuracy                           0.71        68
   macro avg       0.68      0.66      0.66        68
weighted avg       0.70      0.71      0.70        68

[[36  7]
 [13 12]]

C:\Users\christoph.wuersch\.conda\envs\ML\lib\site-packages\sklearn\base.py:450: UserWarning: X does not have valid feature names, but SVC was fitted with feature names
  warnings.warn(

print(X_names.values)

Xtrain_2D=Xtrain.loc[:,['TwoHourSerIns','Age']]
Xtrain_2D

['NumTimesPrg' 'PlGlcConc' 'BloodP' 'SkinThick' 'TwoHourSerIns' 'BMI'
 'DiPedFunc' 'Age']

def PlotDecisionBoundary(model, X2D, y):

    gamma=model.gamma
    C=model.C
    x1=X2D.iloc[:,0].values
    x2=X2D.iloc[:,1].values
        
    # evaluate decision function in a grid
    xx, yy = np.meshgrid(np.linspace(-2, 2, 200), np.linspace(-2, 2, 200))
    
    Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)

    # visualize decision function for these parameters
    
    plt.title("gamma=10^%d, C=10^%d" % (np.log10(gamma), np.log10(C)),
            size='medium')

    # visualize parameter's effect on decision function
    plt.pcolormesh(xx, yy, -Z, cmap=plt.cm.RdBu,shading='nearest')
    plt.contour(xx, yy, -Z,[-1, 0, 1])
    plt.scatter(x1, x2, c=y, cmap=plt.cm.RdBu_r, edgecolors='k')
    plt.xlim([-2,2])
    plt.ylim([-2,2])

# Now we need to fit a classifier for all parameters in the 2d version
# (we use a smaller set of parameters here because it takes a while to train)

C_range = [1e-2, 1, 1e2]
gamma_range = [1e-1, 1, 1e1]
classifiers = []
for C in C_range:
    for gamma in gamma_range:
        clf = SVC(kernel='rbf', degree=2, C=C, gamma=gamma)
        clf.fit(Xtrain_2D.values, ytrain)
        classifiers.append((C, gamma, clf))

# #############################################################################
# Visualization
#
# draw visualization of parameter effects


for (k, (C, gamma, clf)) in enumerate(classifiers):

    plt.figure(figsize=(8, 8))
    # visualize decision function for these parameters
    # plt.subplot(len(C_range), len(gamma_range), k + 1)
    PlotDecisionBoundary(clf, Xtrain_2D, ytrain)
#plt.savefig('diabetes.pdf')

import random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

def generate_random_dataset(size):
    """ Generate a random dataset and that follows a quadratic  distribution
    """
    x = []
    y = []
    target = []
    for i in range(size):
        # class zero
        x.append(np.round(random.uniform(0, 2.5), 1))
        y.append(np.round(random.uniform(0, 20), 1))
        target.append(0)
        # class one
        x.append(np.round(random.uniform(1, 5), 2))
        y.append(np.round(random.uniform(20, 25), 2))
        target.append(1)
        x.append(np.round(random.uniform(3, 5), 2))
        y.append(np.round(random.uniform(5, 25), 2))
        target.append(1)
    df_x = pd.DataFrame(data=x)
    df_y = pd.DataFrame(data=y)
    df_target = pd.DataFrame(data=target)
    data_frame = pd.concat([df_x, df_y], ignore_index=True, axis=1)
    data_frame = pd.concat([data_frame, df_target], ignore_index=True, axis=1)
    data_frame.columns = ['x', 'y', 'target']
    return data_frame

# Generate dataset
size = 100
dataset = generate_random_dataset(size)

X = dataset[['x', 'y']]
y = dataset['target']
dataset.to_csv('dataset.csv')

dataset.head()

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Plotting the training set
fig, ax = plt.subplots(figsize=(12, 7))
# adding major gridlines
ax.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)

ax.scatter(x_train.x, x_train.y, c=y_train)

plt.show()

from sklearn import svm
model = svm.SVC(kernel='poly', degree=2,C=1,gamma=0.10)
model.fit(x_train, y_train)

SVC(C=1, degree=2, gamma=0.1, kernel='poly')

# Plot the dataset
def PlotDecisionBoundary(model, X,y):

    #plot decision boundary for model in case of 2D feature space
    x1=X[:,0]
    x2=X[:,1]
    # Create grid to evaluate model
    xx = np.linspace(min(x1), max(x1), len(x1))
    yy = np.linspace(min(x1), max(x2), len(x2))
    YY, XX = np.meshgrid(yy, xx)
    xy = np.vstack([XX.ravel(), YY.ravel()]).T

    train_size = len(x1)

    # Assigning different colors to the classes
    colors = y
    colors = np.where(colors == 1, '#8C7298', '#4786D1')

    # Get the separating hyperplane
    Z = model.decision_function(xy).reshape(XX.shape)

    fig, ax = plt.subplots(figsize=(12, 7))
    ax.scatter(x1, x2, c=colors)

    # Draw the decision boundary and margins
    ax.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5, linestyles=['--', '-', '--'])

    # Highlight support vectors with a circle around them
    ax.scatter(model.support_vectors_[:, 0], model.support_vectors_[:, 1], s=100,
               linewidth=1, facecolors='none', edgecolors='k')
    plt.title('g=%f | C=%f' % (model.gamma,model.C))
    #plt.savefig('PolynomialKernel.pdf')
    plt.show()

PlotDecisionBoundary(model, x_train.values,y_train)

gammaList=np.logspace(-2,2,5)

for gamma in gammaList:
    model = svm.SVC(kernel='poly', degree=2, C=1, gamma=gamma)
    model.fit(x_train, y_train)
    PlotDecisionBoundary(model, x_train.values,y_train)

gamma=0.001
CList=np.logspace(0,4,5)

for C in CList:
    model = svm.SVC(kernel='rbf',C=C,gamma=gamma)
    model.fit(x_train, y_train)
    PlotDecisionBoundary(model, x_train.values,y_train)

import numpy as np

from matplotlib import pyplot as plt

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import (RBF, Matern, RationalQuadratic,
                                              ExpSineSquared, DotProduct,
                                              ConstantKernel)

kernels = [1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)),
           1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1),
           1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0,
                                length_scale_bounds=(0.1, 10.0),
                                periodicity_bounds=(1.0, 10.0)),
           ConstantKernel(0.1, (0.01, 10.0))
               * (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.0, 10.0)) ** 2),
           1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0),
                        nu=1.5)]

for fig_index, kernel in enumerate(kernels):
    
    # Specify Gaussian Process
    gp = GaussianProcessRegressor(kernel=kernel)

    # Plot prior
    plt.figure(fig_index, figsize=(8, 8))
    plt.subplot(2, 1, 1)
    X_ = np.linspace(0, 5, 100)
    y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
    plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
    plt.fill_between(X_, y_mean - y_std, y_mean + y_std,
                     alpha=0.2, color='k')
    y_samples = gp.sample_y(X_[:, np.newaxis], 10)
    plt.plot(X_, y_samples, lw=1)
    plt.xlim(0, 5)
    plt.ylim(-3, 3)
    plt.title("Prior (kernel:  %s)" % kernel, fontsize=12)

    # Generate data and fit GP
    rng = np.random.RandomState(4)
    X = rng.uniform(0, 5, 10)[:, np.newaxis]
    y = np.sin((X[:, 0] - 2.5) ** 2)
    gp.fit(X, y)

    # Plot posterior
    plt.subplot(2, 1, 2)
    X_ = np.linspace(0, 5, 100)
    y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
    plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
    plt.fill_between(X_, y_mean - y_std, y_mean + y_std,
                     alpha=0.2, color='k')

    y_samples = gp.sample_y(X_[:, np.newaxis], 10)
    plt.plot(X_, y_samples, lw=1)
    plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0))
    plt.xlim(0, 5)
    plt.ylim(-3, 3)
    plt.title("Posterior (kernel: %s)\n Log-Likelihood: %.3f"
              % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)),
              fontsize=12)
    plt.tight_layout()
    plt.show()

C:\Users\christoph.wuersch\.conda\envs\ML\lib\site-packages\sklearn\gaussian_process\kernels.py:430: ConvergenceWarning: The optimal value found for dimension 0 of parameter k2__alpha is close to the specified upper bound 100000.0. Increasing the bound and calling fit again may find a better value.
  warnings.warn(

C:\Users\christoph.wuersch\.conda\envs\ML\lib\site-packages\sklearn\gaussian_process\kernels.py:335: RuntimeWarning: divide by zero encountered in log
  return np.log(np.vstack(bounds))
C:\Users\christoph.wuersch\.conda\envs\ML\lib\site-packages\sklearn\gaussian_process\_gpr.py:616: ConvergenceWarning: lbfgs failed to converge (status=2):
ABNORMAL_TERMINATION_IN_LNSRCH.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
  _check_optimize_result("lbfgs", opt_res)

import numpy as np
import matplotlib.pyplot as plt


A = 1         #amplitude
p = 2         #periodicity
sigma0=0.001    #noise level
N = 200;      #number of points



noise = np.random.normal(0,0.1,N)
X=np.linspace(0,10,N)
y=A*np.sin(2*np.pi*(X/p))+noise
X=X[:, np.newaxis]
y=y[:, np.newaxis]
#generate noisy sine wave

plt.figure()
plt.plot(X,y)
plt.grid(True)

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels \
    import RBF, WhiteKernel, RationalQuadratic, ExpSineSquared

# variation of the amplitude of the sine
k1 = A**2 * RBF(length_scale=30)                         
# periodic component with variable amplitude
k2 = ExpSineSquared(length_scale=20, periodicity=p)   
# noise terms
k3 = WhiteKernel(noise_level=sigma0**2)  

kernel_gpml = k1*k2 + k3


gp = GaussianProcessRegressor(kernel=kernel_gpml,
                              alpha=0.0,
                              optimizer=None,
                              normalize_y=True)
gp.fit(X, y)

GaussianProcessRegressor(alpha=0.0,
                         kernel=1**2 * RBF(length_scale=30) * ExpSineSquared(length_scale=20, periodicity=2) + WhiteKernel(noise_level=1e-06),
                         normalize_y=True, optimizer=None)

GaussianProcessRegressor(alpha=0.0,
                         kernel=1**2 * RBF(length_scale=30) * ExpSineSquared(length_scale=20, periodicity=2) + WhiteKernel(noise_level=1e-06),
                         normalize_y=True, optimizer=None)

X_ = np.linspace(X.min(), X.max() + 10, 100)
X_ = X_[:,np.newaxis]
y_pred, y_std = gp.predict(X_, return_std=True)

# Illustration
plt.scatter(X, y, c='k')
plt.plot(X_, y_pred)
plt.fill_between(X_[:,0], y_pred - 3*y_std,
                 y_pred + 3*y_std,
                 alpha=0.8, color='k')
plt.xlim(X_.min(), X_.max())
plt.xlabel("x")
plt.ylabel('noisy periodic signal')
plt.title('fitting a noisy periodic signal using $\mathcal{GP}$')
plt.tight_layout()
plt.grid(True)
plt.show()

np.shape(X_[:,0])

(100,)

%matplotlib inline

import numpy as np

from matplotlib import pyplot as plt
from matplotlib.colors import LogNorm

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process import kernels


# Generate noisy sine wave
nSamples=300;
rng = np.random.RandomState(0)
X = rng.uniform(0, 5, nSamples)[:, np.newaxis]
y = 0.5 * np.sin(3 * X[:, 0]) + rng.normal(0, 0.3, X.shape[0])

plt.figure(figsize=(10,10))
plt.scatter(X,y)
plt.grid(True)
plt.xlabel('X')
plt.ylabel('X')
plt.title('raw noisy data')

Text(0.5, 1.0, 'raw noisy data')

# First run: using the RBF kernel and white noise

kernel = 1.0 * kernels.RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e3)) \
    + kernels.WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-10, 1e+1))
gp1 = GaussianProcessRegressor(kernel=kernel, alpha=1e-5).fit(X, y)

X_ = np.linspace(0, 10, 100)
y_mean, y_cov = gp1.predict(X_[:, np.newaxis], return_cov=True)

plt.figure(figsize=(10,10))
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)),
                 y_mean + np.sqrt(np.diag(y_cov)),
                 alpha=0.5, color='k')
plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9)
plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0))
plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s"
          % (kernel, gp1.kernel_,
             gp1.log_marginal_likelihood(gp1.kernel_.theta)))
plt.tight_layout()
y_cov.shape

plt.figure()
plt.matshow(y_cov)
plt.show()

<Figure size 432x288 with 0 Axes>

#Get out the hyperparameters
gp1.kernel_

0.426**2 * RBF(length_scale=0.53) + WhiteKernel(noise_level=0.0896)

# Second run: using the sin-exponential kernel and white noise
kernel = 0.5 * kernels.ExpSineSquared(length_scale=4.0,  periodicity=1,
                                      length_scale_bounds=(1e-1, 1e3),
                                     periodicity_bounds=(1e-1, 4)) \
    + kernels.WhiteKernel(noise_level=1e-6, noise_level_bounds=(1e-10, 1e+1))
gp = GaussianProcessRegressor(kernel=kernel, alpha=1E-5).fit(X, y)

X_ = np.linspace(0, 10, 100)
y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True)


plt.figure(figsize=(10,10))
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)),
                 y_mean + np.sqrt(np.diag(y_cov)),
                 alpha=0.5, color='k')
plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9)
plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0))
plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s"
          % (kernel, gp.kernel_,
             gp.log_marginal_likelihood(gp.kernel_.theta)))
plt.tight_layout()

plt.figure()
plt.matshow(y_cov)
plt.show()

<Figure size 432x288 with 0 Axes>

#Get out the hyperparameters
gp.kernel_

0.44**2 * ExpSineSquared(length_scale=0.769, periodicity=4) + WhiteKernel(noise_level=0.0958)

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()

rng = np.random.RandomState(1)
X = np.dot(rng.rand(2, 2), rng.randn(2, 200)).T
plt.scatter(X[:, 0], X[:, 1])
plt.axis('equal');

n=np.shape(X)[0]
C=np.dot(X.T,X)/(n-1)
C

array([[0.68330628, 0.23079731],
       [0.23079731, 0.09884853]])

# eigenvectors and eigenvalues from the scatter matrix
eig_val_sc, eig_vec_sc = np.linalg.eig(C)
eig_val_sc

array([0.76345505, 0.01869975])

eig_vec_sc

array([[ 0.94465994, -0.3280512 ],
       [ 0.3280512 ,  0.94465994]])

from sklearn.decomposition import PCA
pca = PCA(n_components=0.9)
pca.fit(X)

PCA(n_components=0.9)

PCA(n_components=0.9)

pca.n_components_

1

print(pca.components_)

[[-0.94446029 -0.32862557]]

print(pca.explained_variance_ratio_)

[0.97634101]

def draw_vector(v0, v1, ax=None):
    ax = ax or plt.gca()
    arrowprops=dict(arrowstyle='->',
                    linewidth=3,
                    color='k',
                    shrinkA=0, shrinkB=0)
    ax.annotate('', v1, v0, arrowprops=arrowprops)

# plot data
plt.figure(figsize=(8,8))
plt.scatter(X[:, 0], X[:, 1], alpha=0.2)
for length, vector in zip(pca.explained_variance_, pca.components_):
    v = vector * 3 * np.sqrt(length)
    draw_vector(pca.mean_, pca.mean_ + v)
plt.axis('equal');

pca = PCA(n_components=1)
pca.fit(X)
X_pca = pca.transform(X)
print("original shape:   ", X.shape)
print("transformed shape:", X_pca.shape)

original shape:    (200, 2)
transformed shape: (200, 1)

X_new = pca.inverse_transform(X_pca)
plt.scatter(X[:, 0], X[:, 1], alpha=0.2)
plt.scatter(X_new[:, 0], X_new[:, 1], alpha=0.8)
plt.axis('equal');

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.naive_bayes import GaussianNB
from sklearn import metrics
import matplotlib.pyplot as plt
from sklearn.datasets import load_wine
from sklearn.pipeline import make_pipeline

RANDOM_STATE = 42
FIG_SIZE = (10, 7)

features, target = load_wine(return_X_y=True)
target

array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1,
       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2,
       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
       2, 2])

# Make a train/test split using 30% test size
X_train, X_test, y_train, y_test = train_test_split(features, target,
                                                    test_size=0.30,
                                                    random_state=RANDOM_STATE)

# Fit to data and predict using pipelined GNB and PCA.
unscaled_clf = make_pipeline(PCA(n_components=2), GaussianNB())
unscaled_clf.fit(X_train, y_train)
pred_test = unscaled_clf.predict(X_test)

# Show prediction accuracy in unscaled data.
print('\nPrediction accuracy for the normal test dataset with PCA')
print('{:.2%}\n'.format(metrics.accuracy_score(y_test, pred_test)))

Prediction accuracy for the normal test dataset with PCA
81.48%

unscaled_clf = make_pipeline(PCA(n_components=4), GaussianNB())
unscaled_clf.fit(X_train, y_train)
pred_test = unscaled_clf.predict(X_test)

# Show prediction accuracy in unscaled data.
print('\nPrediction accuracy for the normal test dataset with PCA')
print('{:.2%}\n'.format(metrics.accuracy_score(y_test, pred_test)))

Prediction accuracy for the normal test dataset with PCA
98.15%

# Fit to data and predict using pipelined scaling, GNB and PCA.
std_clf = make_pipeline(StandardScaler(), PCA(n_components=2), GaussianNB())
std_clf.fit(X_train, y_train)
pred_test_std = std_clf.predict(X_test)

# Show prediction accuracies in scaled and unscaled data.
print('\nPrediction accuracy for the normal test dataset with PCA')
print('{:.2%}\n'.format(metrics.accuracy_score(y_test, pred_test_std)))

Prediction accuracy for the normal test dataset with PCA
98.15%

# Extract PCA from pipeline
pca = unscaled_clf.named_steps['pca']
X_train_unscaled=pca.transform(X_train)
pca_std = std_clf.named_steps['pca']

# Show first principal componenets
print('\nPC 1 without scaling:\n', pca.components_[0])
print('\nPC 1 with scaling:\n', pca_std.components_[0])

PC 1 without scaling:
 [ 1.76342917e-03 -8.35544737e-04  1.54623496e-04 -5.31136096e-03
  2.01663336e-02  1.02440667e-03  1.53155502e-03 -1.11663562e-04
  6.31071580e-04  2.32645551e-03  1.53606718e-04  7.43176482e-04
  9.99775716e-01]

PC 1 with scaling:
 [ 0.13443023 -0.25680248 -0.0113463  -0.23405337  0.15840049  0.39194918
  0.41607649 -0.27871336  0.33129255 -0.11383282  0.29726413  0.38054255
  0.27507157]

# Scale and use PCA on X_train data for visualization.

scaler = std_clf.named_steps['standardscaler']
X_train_std = pca_std.transform(scaler.transform(X_train))

# visualize standardized vs. untouched dataset with PCA performed
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=FIG_SIZE)


for l, c, m in zip(range(0, 3), ('blue', 'red', 'green'), ('^', 's', 'o')):
    ax1.scatter(X_train_unscaled[y_train == l, 0], X_train_unscaled[y_train == l, 1],
                color=c,
                label='class %s' % l,
                alpha=0.5,
                marker=m
                )

for l, c, m in zip(range(0, 3), ('blue', 'red', 'green'), ('^', 's', 'o')):
    ax2.scatter(X_train_std[y_train == l, 0], X_train_std[y_train == l, 1],
                color=c,
                label='class %s' % l,
                alpha=0.5,
                marker=m
                )

ax1.set_title('Training dataset after PCA')
ax2.set_title('Standardized training dataset after PCA')

for ax in (ax1, ax2):
    ax.set_xlabel('1st principal component')
    ax.set_ylabel('2nd principal component')
    ax.legend(loc='upper right')
    ax.grid()

plt.tight_layout()

plt.show()

# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "dim_reduction"

def save_fig(fig_id, tight_layout=True):
    path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png")
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format='png', dpi=600)

from sklearn.decomposition import PCA

from sklearn.datasets import fetch_openml
mnist = fetch_openml('mnist_784')

#custom_data_home='C:/temp'
#mnist = fetch_openml('mnist_784', data_home=custom_data_home)
mnist.DESCR

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\datasets\_openml.py:1002: FutureWarning: The default value of `parser` will change from `'liac-arff'` to `'auto'` in 1.4. You can set `parser='auto'` to silence this warning. Therefore, an `ImportError` will be raised from 1.4 if the dataset is dense and pandas is not installed. Note that the pandas parser may return different data types. See the Notes Section in fetch_openml's API doc for details.
  warn(

"**Author**: Yann LeCun, Corinna Cortes, Christopher J.C. Burges  \n**Source**: [MNIST Website](http://yann.lecun.com/exdb/mnist/) - Date unknown  \n**Please cite**:  \n\nThe MNIST database of handwritten digits with 784 features, raw data available at: http://yann.lecun.com/exdb/mnist/. It can be split in a training set of the first 60,000 examples, and a test set of 10,000 examples  \n\nIt is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image. It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. The original black and white (bilevel) images from NIST were size normalized to fit in a 20x20 pixel box while preserving their aspect ratio. The resulting images contain grey levels as a result of the anti-aliasing technique used by the normalization algorithm. the images were centered in a 28x28 image by computing the center of mass of the pixels, and translating the image so as to position this point at the center of the 28x28 field.  \n\nWith some classification methods (particularly template-based methods, such as SVM and K-nearest neighbors), the error rate improves when the digits are centered by bounding box rather than center of mass. If you do this kind of pre-processing, you should report it in your publications. The MNIST database was constructed from NIST's NIST originally designated SD-3 as their training set and SD-1 as their test set. However, SD-3 is much cleaner and easier to recognize than SD-1. The reason for this can be found on the fact that SD-3 was collected among Census Bureau employees, while SD-1 was collected among high-school students. Drawing sensible conclusions from learning experiments requires that the result be independent of the choice of training set and test among the complete set of samples. Therefore it was necessary to build a new database by mixing NIST's datasets.  \n\nThe MNIST training set is composed of 30,000 patterns from SD-3 and 30,000 patterns from SD-1. Our test set was composed of 5,000 patterns from SD-3 and 5,000 patterns from SD-1. The 60,000 pattern training set contained examples from approximately 250 writers. We made sure that the sets of writers of the training set and test set were disjoint. SD-1 contains 58,527 digit images written by 500 different writers. In contrast to SD-3, where blocks of data from each writer appeared in sequence, the data in SD-1 is scrambled. Writer identities for SD-1 is available and we used this information to unscramble the writers. We then split SD-1 in two: characters written by the first 250 writers went into our new training set. The remaining 250 writers were placed in our test set. Thus we had two sets with nearly 30,000 examples each. The new training set was completed with enough examples from SD-3, starting at pattern # 0, to make a full set of 60,000 training patterns. Similarly, the new test set was completed with SD-3 examples starting at pattern # 35,000 to make a full set with 60,000 test patterns. Only a subset of 10,000 test images (5,000 from SD-1 and 5,000 from SD-3) is available on this site. The full 60,000 sample training set is available.\n\nDownloaded from openml.org."

np.random.seed(42)

m = 5000
idx = np.random.permutation(60000)[:m]

X = mnist.data.iloc[idx,:].values
y = mnist.target.iloc[idx].values

y[0:5]

['7', '3', '8', '9', '3']
Categories (10, object): ['0', '1', '2', '3', ..., '6', '7', '8', '9']

#we save the data to disk so that we can fetch it fast if we need it using the next cell:

import pandas as pd
mnist.data.to_pickle('mnist_data_784.pkl')
mnist.target.to_pickle('mnist_label_784.pkl');

#read data from pickle file if there is no internet connection
# Xp=pd.read_pickle('mnist_data_784.pkl').values
# yp=pd.read_pickle('mnist_label_784.pkl').values

# m = 5000
# idx = np.random.permutation(60000)[:m]
# X = Xp[idx,:]
# y = yp[idx]

from sklearn.manifold import TSNE

tsne = TSNE(n_components=2, random_state=42)
X_reduced = tsne.fit_transform(X)

plt.figure(figsize=(13,10))
plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=y.astype(int), cmap="jet")
plt.axis('off')
plt.colorbar()
plt.show()

plt.figure(figsize=(8,8))
cmap = matplotlib.cm.get_cmap("jet")
for digit in (2, 3, 5):
    color=np.array(cmap(digit / 9)).reshape(1,4)
    plt.scatter(X_reduced[y.astype(int) == digit, 0], X_reduced[y.astype(int) == digit, 1], c=color,alpha=0.5)
#plt.axis('off')
plt.show()

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4253574472.py:2: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

y

['7', '3', '8', '9', '3', ..., '0', '4', '7', '4', '9']
Length: 5000
Categories (10, object): ['0', '1', '2', '3', ..., '6', '7', '8', '9']

idx = (y == '2') | (y == '3') | (y == '5')
X_subset = X[idx]
y_subset = y[idx]

tsne_subset = TSNE(n_components=2, random_state=42)
X_subset_reduced = tsne_subset.fit_transform(X_subset)

plt.figure(figsize=(9,9))
for digit in (2, 3, 5):
    color=np.array(cmap(digit / 9)).reshape(1,4)
    plt.scatter(X_subset_reduced[y_subset.astype(int) == digit, 0],
                X_subset_reduced[y_subset.astype(int) == digit, 1], c=color)
plt.axis('off')
plt.show()

from sklearn.preprocessing import MinMaxScaler
from matplotlib.offsetbox import AnnotationBbox, OffsetImage

def plot_digits(X, y, min_distance=0.05, images=None, figsize=(13, 10)):
    # Let's scale the input features so that they range from 0 to 1
    X_normalized = MinMaxScaler().fit_transform(X)
    # Now we create the list of coordinates of the digits plotted so far.
    # We pretend that one is already plotted far away at the start, to
    # avoid `if` statements in the loop below
    neighbors = np.array([[10., 10.]])
    # The rest should be self-explanatory
    plt.figure(figsize=figsize)
    cmap = matplotlib.cm.get_cmap("jet")
    digits = np.unique(y)
    for digit in digits:
        color=np.array(cmap(digit / 9)).reshape(1,4)
        plt.scatter(X_normalized[y == digit, 0], X_normalized[y == digit, 1], c=color)
    plt.axis("off")
    ax = plt.gcf().gca()  # get current axes in current figure
    for index, image_coord in enumerate(X_normalized):
        closest_distance = np.linalg.norm(np.array(neighbors) - image_coord, axis=1).min()
        if closest_distance > min_distance:
            neighbors = np.r_[neighbors, [image_coord]]
            if images is None:
                plt.text(image_coord[0], image_coord[1], str(int(y[index])),
                         color=cmap(y[index] / 9), fontdict={"weight": "bold", "size": 16})
            else:
                image = images[index].reshape(28, 28)
                imagebox = AnnotationBbox(OffsetImage(image, cmap="binary"), image_coord)
                ax.add_artist(imagebox)

plot_digits(X_reduced, y.astype(int))

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

plot_digits(X_reduced, y.astype(int), images=X, figsize=(35, 25))

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

plot_digits(X_subset_reduced, y_subset.astype(int), images=X_subset, figsize=(22, 22))

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

from sklearn.decomposition import PCA
import time

t0 = time.time()
X_pca_reduced = PCA(n_components=2, random_state=42).fit_transform(X)
t1 = time.time()
print("PCA took {:.1f}s.".format(t1 - t0))
plot_digits(X_pca_reduced, y.astype(int))
plt.show()

PCA took 0.1s.

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

from sklearn.manifold import LocallyLinearEmbedding

t0 = time.time()
X_lle_reduced = LocallyLinearEmbedding(n_components=2, random_state=42).fit_transform(X)
t1 = time.time()
print("LLE took {:.1f}s.".format(t1 - t0))
plot_digits(X_lle_reduced, y.astype(int))
plt.show()

LLE took 3.4s.

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

from sklearn.pipeline import Pipeline

pca_lle = Pipeline([
    ("pca", PCA(n_components=0.95, random_state=42)),
    ("lle", LocallyLinearEmbedding(n_components=2, random_state=42)),
])
t0 = time.time()
X_pca_lle_reduced = pca_lle.fit_transform(X)
t1 = time.time()
print("PCA+LLE took {:.1f}s.".format(t1 - t0))
plot_digits(X_pca_lle_reduced, y.astype(int))
plt.show()

PCA+LLE took 4.2s.

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

from sklearn.manifold import MDS

m = 2000
t0 = time.time()
X_mds_reduced = MDS(n_components=2, random_state=42).fit_transform(X[:m])
t1 = time.time()
print("MDS took {:.1f}s (on just 2,000 MNIST images instead of 10,000).".format(t1 - t0))
plot_digits(X_mds_reduced, y[:m].astype(int))
plt.show()

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\manifold\_mds.py:298: FutureWarning: The default value of `normalized_stress` will change to `'auto'` in version 1.4. To suppress this warning, manually set the value of `normalized_stress`.
  warnings.warn(

MDS took 185.0s (on just 2,000 MNIST images instead of 10,000).

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

from sklearn.pipeline import Pipeline

pca_mds = Pipeline([
    ("pca", PCA(n_components=0.95, random_state=42)),
    ("mds", MDS(n_components=2, random_state=42)),
])
t0 = time.time()
X_pca_mds_reduced = pca_mds.fit_transform(X[:2000])
t1 = time.time()
print("PCA+MDS took {:.1f}s (on 2,000 MNIST images).".format(t1 - t0))
plot_digits(X_pca_mds_reduced, y[:2000].astype(int))
plt.show()

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\manifold\_mds.py:298: FutureWarning: The default value of `normalized_stress` will change to `'auto'` in version 1.4. To suppress this warning, manually set the value of `normalized_stress`.
  warnings.warn(

PCA+MDS took 189.1s (on 2,000 MNIST images).

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

t0 = time.time()
X_lda_reduced = LinearDiscriminantAnalysis(n_components=2).fit_transform(X, y)
t1 = time.time()
print("LDA took {:.1f}s.".format(t1 - t0))
plot_digits(X_lda_reduced, y.astype(int), figsize=(12,12))
plt.show()

LDA took 0.8s.

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

from sklearn.manifold import TSNE

t0 = time.time()
X_tsne_reduced = TSNE(n_components=2, random_state=42).fit_transform(X)
t1 = time.time()
print("t-SNE took {:.1f}s.".format(t1 - t0))
plot_digits(X_tsne_reduced, y.astype(int))
plt.show()

t-SNE took 31.0s.

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

pca_tsne = Pipeline([
    ("pca", PCA(n_components=0.95, random_state=42)),
    ("tsne", TSNE(n_components=2, random_state=42)),
])
t0 = time.time()
X_pca_tsne_reduced = pca_tsne.fit_transform(X)
t1 = time.time()
print("PCA+t-SNE took {:.1f}s.".format(t1 - t0))
plot_digits(X_pca_tsne_reduced, y.astype(int))
plt.show()

PCA+t-SNE took 33.6s.

C:\Users\colin\AppData\Local\Temp\ipykernel_10168\4093154656.py:13: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
  cmap = matplotlib.cm.get_cmap("jet")

# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "dim_reduction"

def save_fig(fig_id, tight_layout=True):
    path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png")
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format='png', dpi=300)

from sklearn.decomposition import PCA

#from sklearn.datasets import fetch_mldata

from sklearn.datasets import fetch_openml
mnist = fetch_openml('mnist_784')

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\datasets\_openml.py:1002: FutureWarning: The default value of `parser` will change from `'liac-arff'` to `'auto'` in 1.4. You can set `parser='auto'` to silence this warning. Therefore, an `ImportError` will be raised from 1.4 if the dataset is dense and pandas is not installed. Note that the pandas parser may return different data types. See the Notes Section in fetch_openml's API doc for details.
  warn(

X_train = mnist['data'][:60000].values
y_train = mnist['target'][:60000].astype(int).values

X_test = mnist['data'][60000:].values
y_test = mnist['target'][60000:].astype(int).values

X_train
y_train

array([5, 0, 4, ..., 5, 6, 8])

#we save the data to disk so that we can fetch it fast if we need it using the next cell:

import pandas as pd
#mnist.data.to_pickle('mnist_data_784.pkl')
#mnist.target.to_pickle('mnist_label_784.pkl');

#read data from pickle file if there is no internet connection
X=pd.read_pickle('mnist_data_784.pkl').values
y=pd.read_pickle('mnist_label_784.pkl').values

X_train = X[:60000:,:]
y_train = y[:60000].astype(int)

X_test = X[60000:,:]
y_test = y[60000:].astype(int)

from sklearn.ensemble import RandomForestClassifier

rnd_clf = RandomForestClassifier(random_state=42)

import time

t0 = time.time()
rnd_clf.fit(X_train, y_train)
t1 = time.time()

print("Training took {:.2f}s".format(t1 - t0))

Training took 74.57s

from sklearn.metrics import accuracy_score

y_pred = rnd_clf.predict(X_test)
accuracy_score(y_test, y_pred)

0.9705

from sklearn.decomposition import PCA

pca = PCA(n_components=0.95)
X_train_reduced = pca.fit_transform(X_train)

rnd_clf2 = RandomForestClassifier(random_state=42)
t0 = time.time()
rnd_clf2.fit(X_train_reduced, y_train)
t1 = time.time()

print("Training took {:.2f}s".format(t1 - t0))

Training took 320.77s

X_test_reduced = pca.transform(X_test)

y_pred = rnd_clf2.predict(X_test_reduced)
accuracy_score(y_test, y_pred)

0.9481

from sklearn.linear_model import LogisticRegression

log_clf = LogisticRegression(multi_class="multinomial", solver="lbfgs", random_state=42,max_iter=1000)
t0 = time.time()
log_clf.fit(X_train, y_train)
t1 = time.time()

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\linear_model\_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

print("Training took {:.2f}s".format(t1 - t0))

Training took 112.15s

y_pred = log_clf.predict(X_test)
accuracy_score(y_test, y_pred)

0.921

log_clf2 = LogisticRegression(multi_class="multinomial", solver="lbfgs", random_state=42,max_iter=1000)
t0 = time.time()
log_clf2.fit(X_train_reduced, y_train)
t1 = time.time()

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\linear_model\_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

print("Training took {:.2f}s".format(t1 - t0))

Training took 45.11s

y_pred = log_clf2.predict(X_test_reduced)
accuracy_score(y_test, y_pred)

0.9229

import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import load_digits
from sklearn.decomposition import PCA

#digits = load_digits()
#digits.data.shape

from sklearn.datasets import fetch_openml
mnist = fetch_openml('mnist_784',parser='auto')

#digits.data = mnist['data'][:2000]
#digits.target = mnist['target'][:2000].astype(int)

mnist.data

digits=mnist

digits.data = mnist['data'][:2000]
digits.target = mnist['target'][:2000].astype(int)

data=digits.data.iloc[:,:2000].values
target=digits.target.iloc[:2000].values

def plot_digits(data):
    fig, axes = plt.subplots(4, 10, figsize=(10, 4),
                             subplot_kw={'xticks':[], 'yticks':[]},
                             gridspec_kw=dict(hspace=0.1, wspace=0.1))
    for i, ax in enumerate(axes.flat):
        ax.imshow(data[i].reshape(28, 28),
                  cmap='binary', interpolation='nearest',
                  clim=(0, 256))
plot_digits(data)

np.random.seed(42)
noisy = np.random.normal(data, 40)
plot_digits(noisy)

pca = PCA(0.50).fit(noisy)
pca.n_components_

22

components = pca.transform(noisy)
filtered = pca.inverse_transform(components)
plot_digits(filtered)

import os
os.environ['OMP_NUM_THREADS'] = '1'

%matplotlib inline

from IPython.display import Image
import matplotlib.pyplot as plt

from sklearn.datasets import make_blobs
import numpy as np

n_samples = 250
n_features = 8
centers = 8
X, y = make_blobs(n_samples=n_samples, 
                  n_features=n_features, 
                  centers=centers, 
                  cluster_std=1*np.array([0.8,0.9,1.2,1,1.0,0.7,0.9,0.9]), 
                  shuffle=True, 
                  random_state=0)

plt.scatter(X[:, 7], X[:, 4], c=y, marker='o', edgecolor='black', s=50)
plt.grid()
plt.tight_layout()
#plt.savefig('images/11_01.png', dpi=300)
plt.show()

from sklearn.cluster import KMeans

km = KMeans(n_clusters=8, 
            init='k-means++', 
            n_init=10, 
            max_iter=300,
            tol=1e-04,
            random_state=0)

y_km = km.fit_predict(X)
print(km.score(X))
print(km.inertia_)

-1590.0542314855682
1590.0542314855682

from matplotlib import colors as mcolors
colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS)
ColorNames=list(colors.keys())
HSV=colors.values()


def PlotClusters(X,y, km):
    print("%i clusters" % km.n_clusters)
    plt.figure()
    for ClusterNumber in range(km.n_clusters):
        plt.scatter(X[y == ClusterNumber, 7],
                X[y == ClusterNumber, 4],
                s=50, c=ColorNames[ClusterNumber+1],
                marker='s', edgecolor='black',
                label='cluster {0}'.format(ClusterNumber+1))
    plt.scatter(km.cluster_centers_[:, 7],
        km.cluster_centers_[:, 4],
        s=250, marker='*',
        c='red', edgecolor='black',
        label='centroids')
    plt.legend(scatterpoints=1)
    plt.grid()
    plt.tight_layout()
    #plt.savefig('images/11_02.png', dpi=300)
    plt.show()

for n_clusters in range(2,9):
    km = KMeans(n_clusters=n_clusters,
                init='random', 
                n_init=10, 
                max_iter=300,
                tol=1e-04,
                random_state=0)

    y_km = km.fit_predict(X)
    PlotClusters(X,y_km, km)

2 clusters

3 clusters

4 clusters

5 clusters

6 clusters

7 clusters

8 clusters

# Important to check which features are used for the x and y coordinates when plotting!

# If points are closer to the centroid of a different cluster compared to the distance to their own centroid,
# then this can be due to the algorythm being in a local minimum. The global minimum is not always found.

# A change in the random seed for the initial centroid placement can lead to different minimums and better results.

print('Distortion: %.2f' % km.inertia_)

Distortion: 5524.97

km.inertia_

5524.96603004677

import numpy as np
distortions = []
ScoreList   = []
InertiaList = []
maxNumberOfClusters=15

for k in range(1, maxNumberOfClusters):
    km = KMeans(n_clusters=k, 
                init='k-means++', 
                n_init=10, 
                max_iter=300, 
                random_state=0)
    km.fit(X)
    distortions.append(km.inertia_)
    BIC=-km.score(X)/np.sqrt(n_samples)+np.log(n_samples)*n_features*k
    ScoreList.append(BIC)
    InertiaList.append(-km.score(X)/np.sqrt(n_samples))
   
    
#plt.semilogy(range(1, maxNumberOfClusters), distortions, marker='o')
plt.plot(range(1, maxNumberOfClusters), ScoreList, marker='^',label='BIC')
plt.plot(range(1, maxNumberOfClusters), InertiaList, marker='o',label='inertia')
plt.xlabel('Number of clusters')
plt.ylabel('Distortion')
plt.legend()
plt.tight_layout()
plt.grid(True)
#plt.savefig('images/11_03.png', dpi=300)
plt.show()

for n_clusters in range(4,9):
    km = KMeans(n_clusters=n_clusters,
                init='k-means++', 
                n_init=10, 
                max_iter=300,
                tol=1e-04,
                random_state=0)

    y_km = km.fit_predict(X)
    PlotClusters(X,y, km)

4 clusters

5 clusters

6 clusters

7 clusters

8 clusters

import numpy as np
from matplotlib import cm
from sklearn.metrics import silhouette_samples

km = KMeans(n_clusters=4, 
            init='k-means++', 
            n_init=10, 
            max_iter=300,
            tol=1e-04,
            random_state=0)
y_km = km.fit_predict(X)

cluster_labels = np.unique(y_km)
n_clusters = cluster_labels.shape[0]
silhouette_vals = silhouette_samples(X, y_km, metric='euclidean')
y_ax_lower, y_ax_upper = 0, 0
yticks = []
for i, c in enumerate(cluster_labels):
    c_silhouette_vals = silhouette_vals[y_km == c]
    c_silhouette_vals.sort()
    y_ax_upper += len(c_silhouette_vals)
    color = cm.jet(float(i) / n_clusters)
    plt.barh(range(y_ax_lower, y_ax_upper), c_silhouette_vals, height=1.0, 
             edgecolor='none', color=color)

    yticks.append((y_ax_lower + y_ax_upper) / 2.)
    y_ax_lower += len(c_silhouette_vals)
    
silhouette_avg = np.mean(silhouette_vals)
plt.axvline(silhouette_avg, color="red", linestyle="--") 

plt.yticks(yticks, cluster_labels + 1)
plt.ylabel('Cluster')
plt.xlabel('Silhouette coefficient')

plt.tight_layout()
#plt.savefig('images/11_04.png', dpi=300)
plt.show()
print(silhouette_avg)

0.5136457374680606

km = KMeans(n_clusters=8,
            init='k-means++',
            n_init=10,
            max_iter=300,
            tol=1e-04,
            random_state=0)
y_km = km.fit_predict(X)

PlotClusters(X,y_km, km)

8 clusters

import numpy as np

cluster_labels = np.unique(y_km)
n_clusters = cluster_labels.shape[0]
silhouette_vals = silhouette_samples(X, y_km, metric='euclidean')
y_ax_lower, y_ax_upper = 0, 0
yticks = []
for i, c in enumerate(cluster_labels):
    c_silhouette_vals = silhouette_vals[y_km == c]
    c_silhouette_vals.sort()
    y_ax_upper += len(c_silhouette_vals)
    color = cm.jet(float(i) / n_clusters)
    plt.barh(range(y_ax_lower, y_ax_upper), c_silhouette_vals, height=1.0, 
             edgecolor='none', color=color)

    yticks.append((y_ax_lower + y_ax_upper) / 2.)
    y_ax_lower += len(c_silhouette_vals)
    
silhouette_avg = np.mean(silhouette_vals)
plt.axvline(silhouette_avg, color="red", linestyle="--") 

plt.yticks(yticks, cluster_labels + 1)
plt.ylabel('Cluster')
plt.xlabel('Silhouette coefficient')

plt.tight_layout()
#plt.savefig('images/11_06.png', dpi=300)
plt.show()

Image(filename='./images/11_05.png', width=400)

import pandas as pd
import numpy as np

np.random.seed(123)

variables = ['X', 'Y', 'Z']
labels = ['ID_0', 'ID_1', 'ID_2', 'ID_3', 'ID_4']

X = np.random.random_sample([5, 3])*10
df = pd.DataFrame(X, columns=variables, index=labels)
df

from scipy.spatial.distance import pdist, squareform

row_dist = pd.DataFrame(squareform(pdist(df, metric='euclidean')),
                        columns=labels,
                        index=labels)
row_dist

# 1. incorrect approach: Squareform distance matrix

from scipy.cluster.hierarchy import linkage

row_clusters = linkage(row_dist, method='complete', metric='euclidean')
pd.DataFrame(row_clusters,
             columns=['row label 1', 'row label 2',
                      'distance', 'no. of items in clust.'],
             index=['cluster %d' % (i + 1)
                    for i in range(row_clusters.shape[0])])

C:\Users\colin\AppData\Local\Temp\ipykernel_7320\2571863248.py:5: ClusterWarning: scipy.cluster: The symmetric non-negative hollow observation matrix looks suspiciously like an uncondensed distance matrix
  row_clusters = linkage(row_dist, method='complete', metric='euclidean')

# 2. correct approach: Condensed distance matrix

row_clusters = linkage(pdist(df, metric='euclidean'), method='complete')
pd.DataFrame(row_clusters,
             columns=['row label 1', 'row label 2',
                      'distance', 'no. of items in clust.'],
             index=['cluster %d' % (i + 1) 
                    for i in range(row_clusters.shape[0])])

# 3. correct approach: Input sample matrix

row_clusters = linkage(df.values, method='complete', metric='euclidean')

pd.DataFrame(row_clusters,
             columns=['row label 1', 'row label 2',
                      'distance', 'no. of items in clust.'],
             index=['cluster %d' % (i + 1)
                    for i in range(row_clusters.shape[0])])

from scipy.cluster.hierarchy import dendrogram

# make dendrogram black (part 1/2)
# from scipy.cluster.hierarchy import set_link_color_palette
# set_link_color_palette(['black'])

row_dendr = dendrogram(row_clusters, 
                       labels=labels,
                       # make dendrogram black (part 2/2)
                       # color_threshold=np.inf
                       )
plt.tight_layout()
plt.ylabel('Euclidean distance')
#plt.savefig('images/11_11.png', dpi=300, 
#            bbox_inches='tight')
plt.show()

# plot row dendrogram
fig = plt.figure(figsize=(8, 8), facecolor='white')
axd = fig.add_axes([0.09, 0.1, 0.2, 0.6])

# note: for matplotlib < v1.5.1, please use orientation='right'
row_dendr = dendrogram(row_clusters, orientation='left')

# reorder data with respect to clustering
df_rowclust = df.iloc[row_dendr['leaves'][::-1]]

axd.set_xticks([])
axd.set_yticks([])

# remove axes spines from dendrogram
for i in axd.spines.values():
    i.set_visible(False)

# plot heatmap
axm = fig.add_axes([0.23, 0.1, 0.6, 0.6])  # x-pos, y-pos, width, height
cax = axm.matshow(df_rowclust, interpolation='nearest', cmap='hot_r')
fig.colorbar(cax)
axm.set_xticklabels([''] + list(df_rowclust.columns))
axm.set_yticklabels([''] + list(df_rowclust.index))

#plt.savefig('images/11_12.png', dpi=300)
plt.show()

C:\Users\colin\AppData\Local\Temp\ipykernel_7320\1264724331.py:22: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  axm.set_xticklabels([''] + list(df_rowclust.columns))
C:\Users\colin\AppData\Local\Temp\ipykernel_7320\1264724331.py:23: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  axm.set_yticklabels([''] + list(df_rowclust.index))

from sklearn.cluster import AgglomerativeClustering

ac = AgglomerativeClustering(n_clusters=3, 
                             affinity='euclidean', 
                             linkage='complete')
labels = ac.fit_predict(X)
print('Cluster labels: %s' % labels)

Cluster labels: [1 0 0 2 1]

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\cluster\_agglomerative.py:1005: FutureWarning: Attribute `affinity` was deprecated in version 1.2 and will be removed in 1.4. Use `metric` instead
  warnings.warn(

ac = AgglomerativeClustering(n_clusters=2, 
                             affinity='euclidean', 
                             linkage='complete')
labels = ac.fit_predict(X)
print('Cluster labels: %s' % labels)

Cluster labels: [0 1 1 0 0]

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\cluster\_agglomerative.py:1005: FutureWarning: Attribute `affinity` was deprecated in version 1.2 and will be removed in 1.4. Use `metric` instead
  warnings.warn(

Image(filename='images/11_13.png', width=500)

from sklearn.datasets import make_moons

X, y = make_moons(n_samples=200, noise=0.1, random_state=0)
plt.scatter(X[:, 0], X[:, 1])
plt.tight_layout()
#plt.savefig('images/11_14.png', dpi=300)
plt.show()

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 3))

km = KMeans(n_clusters=2, random_state=0)
y_km = km.fit_predict(X)
ax1.scatter(X[y_km == 0, 0], X[y_km == 0, 1],
            edgecolor='black',
            c='lightblue', marker='o', s=40, label='cluster 1')
ax1.scatter(X[y_km == 1, 0], X[y_km == 1, 1],
            edgecolor='black',
            c='red', marker='s', s=40, label='cluster 2')
ax1.set_title('K-means clustering')

ac = AgglomerativeClustering(n_clusters=2,
                             affinity='euclidean',
                             linkage='complete')
y_ac = ac.fit_predict(X)
ax2.scatter(X[y_ac == 0, 0], X[y_ac == 0, 1], c='lightblue',
            edgecolor='black',
            marker='o', s=40, label='cluster 1')
ax2.scatter(X[y_ac == 1, 0], X[y_ac == 1, 1], c='red',
            edgecolor='black',
            marker='s', s=40, label='cluster 2')
ax2.set_title('Agglomerative clustering')

plt.legend()
plt.tight_layout()
# plt.savefig('images/11_15.png', dpi=300)
plt.show()

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\cluster\_kmeans.py:1412: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
  super()._check_params_vs_input(X, default_n_init=10)
c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\cluster\_agglomerative.py:1005: FutureWarning: Attribute `affinity` was deprecated in version 1.2 and will be removed in 1.4. Use `metric` instead
  warnings.warn(

from sklearn.cluster import DBSCAN

db = DBSCAN(eps=0.2, min_samples=5, metric='euclidean')
y_db = db.fit_predict(X)
plt.scatter(X[y_db == 0, 0], X[y_db == 0, 1],
            c='lightblue', marker='o', s=40,
            edgecolor='black', 
            label='cluster 1')
plt.scatter(X[y_db == 1, 0], X[y_db == 1, 1],
            c='red', marker='s', s=40,
            edgecolor='black', 
            label='cluster 2')
plt.legend()
plt.tight_layout()
#plt.savefig('images/11_16.png', dpi=300)
plt.show()

y_db

array([ 0,  1,  1, -1,  1,  1,  0,  1,  0,  1,  0,  1,  1,  1,  0,  0,  0,
        1,  0,  0,  1,  1,  0,  1,  0,  1,  1,  1,  1,  0,  0,  0,  1,  1,
        0,  1,  1,  0,  0,  1,  1,  0,  0,  1,  1,  0,  0,  0,  1,  1,  0,
        1,  1,  0,  1,  0,  0,  1,  0,  0,  1,  0,  1,  0,  1,  0,  0,  1,
        0,  0,  1,  0,  1,  0,  1,  0,  1,  0,  0,  1,  1,  0,  1,  1,  1,
        0,  0,  0,  1,  1,  0,  0,  1,  0,  1,  1,  1,  1,  0,  1, -1,  1,
        0,  0,  0,  1,  0,  0,  1,  0,  0,  0,  0,  0,  0,  1,  0,  1,  1,
        0,  0,  0,  1,  0,  1,  0,  0,  1,  1,  1,  0,  0,  0,  1,  1,  1,
        1,  0,  1,  0,  1,  1,  0,  0,  0,  0,  1,  1,  0,  1,  1,  1,  0,
        0,  1,  0,  1,  1,  0,  0,  1,  1,  0,  1,  1,  1,  0,  1,  1,  1,
        0,  0,  0,  0,  1,  1, -1,  0,  0,  0,  1,  0,  1,  1,  1,  0,  0,
        1,  0,  0,  0,  0,  0,  0,  1,  0,  1,  1,  0,  1], dtype=int64)

import warnings
warnings.filterwarnings("ignore") 
from IPython.core.display import display, HTML

import time

import pandas as pd
#import pandas_datareader.data as web
import numpy as np
import scipy.stats as scs
from scipy.stats import multivariate_normal as mvn
import sklearn.mixture as mix

import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

# import class heights
f = 'https://raw.githubusercontent.com/BlackArbsCEO/Mixture_Models/K-Means%2C-E-M%2C-Mixture-Models/Class_heights.csv'

#data = pd.read_csv(f)
#data.to_csv('Class_heights.csv')
# data.info()

data=pd.read_csv('Class_heights.csv',index_col=0)
height = data['Height (in)']
data

# Code sourced from:
#    http://people.duke.edu/~ccc14/sta-663/EMAlgorithm.html

def em_gmm_orig(xs, pis, mus, sigmas, tol=0.01, max_iter=100):

    n, p = xs.shape
    k = len(pis)

    ll_old = 0
    for i in range(max_iter):
        print('\nIteration: ', i)
        print()
        exp_A = []
        exp_B = []
        ll_new = 0

        # E-step
        ws = np.zeros((k, n))
        for j in range(len(mus)):
            for i in range(n):
                ws[j, i] = pis[j] * mvn(mus[j], sigmas[j]).pdf(xs[i])
        ws /= ws.sum(0)

        # M-step
        pis = np.zeros(k)
        for j in range(len(mus)):
            for i in range(n):
                pis[j] += ws[j, i]
        pis /= n

        mus = np.zeros((k, p))
        for j in range(k):
            for i in range(n):
                mus[j] += ws[j, i] * xs[i]
            mus[j] /= ws[j, :].sum()
        
        sigmas = np.zeros((k, p, p))
        for j in range(k):
            for i in range(n):
                ys = np.reshape(xs[i]- mus[j], (2,1))
                sigmas[j] += ws[j, i] * np.dot(ys, ys.T)
            sigmas[j] /= ws[j,:].sum()

        new_mus = (np.diag(mus)[0], np.diag(mus)[1])
        new_sigs = (np.unique(np.diag(sigmas[0]))[0], np.unique(np.diag(sigmas[1]))[0])
        df = (pd.DataFrame(index=[1, 2]).assign(mus = new_mus).assign(sigs = new_sigs))
        
        xx = np.linspace(0, 100, 100)
        yy = scs.multivariate_normal.pdf(xx, mean=new_mus[0], cov=new_sigs[0])
        
        colors = sns.color_palette('Dark2', 3)
        fig, ax = plt.subplots(figsize=(9, 7))
        ax.set_ylim(-0.001, np.max(yy))
        ax.plot(xx, yy, color=colors[1])
        ax.axvline(new_mus[0], ymin=0., color=colors[1])
        ax.fill_between(xx, 0, yy, alpha=0.5, color=colors[1])
        lo, hi = ax.get_ylim()
        ax.annotate(f'$\mu_1$: {new_mus[0]:3.2f}', 
                    fontsize=12, fontweight='demi',
                    xy=(new_mus[0], (hi-lo) / 2), 
                    xycoords='data', xytext=(80, (hi-lo) / 2),
                    arrowprops=dict(facecolor='black', connectionstyle="arc3,rad=0.2",shrink=0.05))
        ax.fill_between(xx, 0, yy, alpha=0.5, color=colors[2])
        
        yy2 = scs.multivariate_normal.pdf(xx, mean=new_mus[1], cov=new_sigs[1])
        
        ax.plot(xx, yy2, color=colors[2])
        ax.axvline(new_mus[1], ymin=0., color=colors[2])
        lo, hi = ax.get_ylim()
        ax.annotate(f'$\mu_2$: {new_mus[1]:3.2f}', 
                    fontsize=12, fontweight='demi',
            xy=(new_mus[1], (hi-lo) / 2), xycoords='data', xytext=(25, (hi-lo) / 2),
            arrowprops=dict(facecolor='black', connectionstyle="arc3,rad=0.2",shrink=0.05))
        ax.fill_between(xx, 0, yy2, alpha=0.5, color=colors[2])
        
        dot_kwds = dict(markerfacecolor='white', markeredgecolor='black', markeredgewidth=1, markersize=10)
        ax.plot(height, len(height)*[0], 'o', **dot_kwds)
        ax.set_ylim(-0.001, np.max(yy2))
        
        
        print(df.T)   
        
        # update complete log likelihoood
        ll_new = 0.0
        for i in range(n):
            s = 0
            for j in range(k):
                s += pis[j] * mvn(mus[j], sigmas[j]).pdf(xs[i])
            ll_new += np.log(s)
        print(f'log_likelihood: {ll_new:3.4f}')
        if np.abs(ll_new - ll_old) < tol:
            break
        ll_old = ll_new
        
    return ll_new, pis, mus, sigmas

height = data['Height (in)']
n = len(height)

# Ground truthish
_mus = np.array([[0, data.groupby('Gender').mean().iat[0, 0]], 
                 [data.groupby('Gender').mean().iat[1, 0], 0]])
_sigmas = np.array([[[5, 0], [0, 5]], 
                    [[5, 0],[0, 5]]])
_pis = np.array([0.5, 0.5]) # priors

# initial random guesses for parameters
np.random.seed(0)

pis = np.random.random(2)
pis /= pis.sum()
mus = np.random.random((2,2))
sigmas = np.array([np.eye(2)] * 2) * height.std()

# generate our noisy x values
xs = np.concatenate([np.random.multivariate_normal(mu, sigma, int(pi*n))
                    for pi, mu, sigma in zip(_pis, _mus, _sigmas)])

ll, pis, mus, sigmas = em_gmm_orig(xs, pis, mus, sigmas)

# In the below plots the white dots represent the observed heights.

Iteration:  0

               1           2
mus    61.362928   59.659685
sigs  469.240750  244.382352
log_likelihood: -141.8092

Iteration:  1

              1          2
mus    68.73773  63.620554
sigs  109.85442   7.228183
log_likelihood: -118.0520

Iteration:  2

              1          2
mus   70.569842  63.688825
sigs   4.424452   3.139277
log_likelihood: -100.2591

Iteration:  3

              1          2
mus   70.569842  63.688825
sigs   4.424427   3.139278
log_likelihood: -100.2591

# Let's define some demo variables and make some blobs

# demo variables

k = 4
n_draws = 500
sigma = .7
random_state = 0
dot_size = 50
cmap = 'viridis'

# make blobs

from sklearn.datasets import make_blobs

X, y_true = make_blobs(n_samples = n_draws,
                       centers = k,
                       cluster_std = sigma,
                       random_state = random_state)

fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(X[:, 0], X[:, 1], s=dot_size)
plt.title('k-means make blobs', fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'k-means make blobs')

# sample implementation
# code sourced from: 
#   http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/05.11-K-Means.ipynb

from sklearn.metrics import pairwise_distances_argmin

def find_clusters(X, n_clusters, rseed=2):
    # 1. Random initialization (choose random clusters)
    rng = np.random.RandomState(rseed)
    i = rng.permutation(X.shape[0])[:n_clusters]
    centers = X[i]

    while True:
        # 2a. Assign labels based on closest center
        labels = pairwise_distances_argmin(X, centers)

        # 2b. Find new centers from means of points
        new_centers = np.array([X[labels == i].mean(0) 
                                for i in range(n_clusters)])

        # 2c. Check for convergence
        if np.all(centers == new_centers):
            break
        centers = new_centers
    
    return centers, labels

# now let's compare this to the sklearn's KMeans() algorithm

# fit k-means to blobs

from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=k)
kmeans.fit(X)
y_kmeans = kmeans.predict(X)

# visualize prediction
fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(X[:, 0], X[:, 1], c=y_kmeans, s=dot_size, cmap=cmap)

# get centers for plot
centers = kmeans.cluster_centers_
ax.scatter(centers[:, 0], centers[:, 1], c='black', s=200, alpha=0.75)
plt.title('sklearn k-means', fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'sklearn k-means')

# let's test the implementation

centers, labels = find_clusters(X, k)

fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(X[:, 0], X[:, 1], c=labels, s=dot_size, cmap=cmap)
plt.title('find_clusters() k-means func', fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'find_clusters() k-means func')

# code sourced from:
#   http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/06.00-Figure-Code.ipynb#Covariance-Type

from ipywidgets import interact

def plot_kmeans_interactive(min_clusters=1, max_clusters=6):
    X, y = make_blobs(n_samples=300, centers=4,
                      random_state=0, cluster_std=0.60)
        
    def plot_points(X, labels, n_clusters):
        plt.scatter(X[:, 0], X[:, 1], c=labels, s=50, cmap='viridis',
                    vmin=0, vmax=n_clusters - 1);
            
    def plot_centers(centers):
        plt.scatter(centers[:, 0], centers[:, 1], marker='o',
                    c=np.arange(centers.shape[0]),
                    s=200, cmap='viridis')
        plt.scatter(centers[:, 0], centers[:, 1], marker='o',
                    c='black', s=50)
            

    def _kmeans_step(frame=0, n_clusters=4):
        rng = np.random.RandomState(2)
        labels = np.zeros(X.shape[0])
        centers = rng.randn(n_clusters, 2)

        nsteps = frame // 3

        for i in range(nsteps + 1):
            old_centers = centers
            if i < nsteps or frame % 3 > 0:
                labels = pairwise_distances_argmin(X, centers)

            if i < nsteps or frame % 3 > 1:
                centers = np.array([X[labels == j].mean(0)
                                    for j in range(n_clusters)])
                nans = np.isnan(centers)
                centers[nans] = old_centers[nans]

        # plot the data and cluster centers
        plot_points(X, labels, n_clusters)
        plot_centers(old_centers)

        # plot new centers if third frame
        if frame % 3 == 2:
            for i in range(n_clusters):
                plt.annotate('', centers[i], old_centers[i], 
                             arrowprops=dict(arrowstyle='->', linewidth=1))
            plot_centers(centers)

        plt.xlim(-4, 4)
        plt.ylim(-2, 10)

        
        if frame % 3 == 1:
            plt.text(3.8, 9.5, "1. Reassign points to nearest centroid",
                     ha='right', va='top', size=14)
        elif frame % 3 == 2:
            plt.text(3.8, 9.5, "2. Update centroids to cluster means",
                     ha='right', va='top', size=14)
    
    return interact(_kmeans_step, frame=[0, 10, 20, 30, 40, 50, 300],
                    n_clusters=np.arange(min_clusters, max_clusters+1))

plot_kmeans_interactive(min_clusters=1, max_clusters=6)

interactive(children=(Dropdown(description='frame', options=(0, 10, 20, 30, 40, 50, 300), value=0), Dropdown(d…

<function __main__.plot_kmeans_interactive.<locals>._kmeans_step(frame=0, n_clusters=4)>

plot_kmeans_interactive(min_clusters=1, max_clusters=6)

interactive(children=(Dropdown(description='frame', options=(0, 10, 20, 30, 40, 50, 300), value=0), Dropdown(d…

<function __main__.plot_kmeans_interactive.<locals>._kmeans_step(frame=0, n_clusters=4)>

centers, labels = find_clusters(X, k, rseed=11)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('sub-optimal clustering', fontsize=18, fontweight='demi')
ax.scatter(X[:, 0], X[:, 1], c=labels, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x29931091c40>

labels6 = KMeans(6, random_state=random_state).fit_predict(X)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('too many clusters', fontsize=18, fontweight='demi')
ax.scatter(X[:, 0], X[:, 1], c=labels6, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x299311408b0>

from sklearn.datasets import make_moons

X_mn, y_mn = make_moons(500, noise=.07, random_state=random_state)

labelsM = KMeans(2, random_state=random_state).fit_predict(X_mn)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('linear separation not possible', fontsize=18, fontweight='demi')
ax.scatter(X_mn[:, 0], X_mn[:, 1], c=labelsM, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x299316f0c10>

from sklearn.cluster import SpectralClustering

model = SpectralClustering(n_clusters=2, affinity='nearest_neighbors',
                          assign_labels='kmeans')

labelsS = model.fit_predict(X_mn)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('kernal transform to higher dimension\nlinear separation is possible', fontsize=18, fontweight='demi')
plt.scatter(X_mn[:, 0], X_mn[:, 1], c=labelsS, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x29931765f40>

# k-means weaknesses that mixture models address directly
# code sourced from:
#   http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/05.12-Gaussian-Mixtures.ipynb

from scipy.spatial.distance import cdist

def plot_kmeans(kmeans, X, n_clusters=k, rseed=2, ax=None):
    labels = kmeans.fit_predict(X)
    
    # plot input data
    #ax = ax or plt.gca() # <-- nice trick
    fig, ax = plt.subplots(figsize=(9,7))    
    ax.axis('equal')
    ax.scatter(X[:, 0], X[:, 1],
               c=labels, s=dot_size, cmap=cmap, zorder=2)
    
    # plot the representation of Kmeans model
    centers = kmeans.cluster_centers_
    radii = [cdist(X[labels==i], [center]).max() 
             for i, center in enumerate(centers)]
    
    for c, r in zip(centers, radii):
        ax.add_patch(plt.Circle(c, r, fc='#CCCCCC',edgecolor='slategrey',
                                lw=4, alpha=0.5, zorder=1))
    return      
                     
X3, y_true = make_blobs(n_samples = 400,
                       centers = k,
                       cluster_std = .6,
                       random_state = random_state)
X3 = X3[:, ::-1] # better plotting

kmeans = KMeans(n_clusters=k, random_state=random_state)
plot_kmeans(kmeans, X3)
plt.title('Clusters are hard circular boundaries', fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'Clusters are hard circular boundaries')

rng = np.random.RandomState(13)
X3_stretched = np.dot(X3, rng.randn(2, 2))

kmeans = KMeans(n_clusters=k, random_state=random_state)
plot_kmeans(kmeans, X3_stretched)
plt.title('Clusters cannot adjust to elliptical data structures',
         fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'Clusters cannot adjust to elliptical data structures')

# code sourced from:
#  http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/05.12-Gaussian-Mixtures.ipynb
from matplotlib.patches import Ellipse 

def draw_ellipse(position, covariance, ax=None, **kwargs):
    """Draw an ellipse with a given position and covariance"""
    
    # Convert covariance to principal axes
    if covariance.shape == (2, 2):
        U, s, Vt = np.linalg.svd(covariance)
        angle = np.degrees(np.arctan2(U[1, 0], U[0, 0]))
        width, height = 2 * np.sqrt(s)
    else:
        angle = 0
        width, height = 2 * np.sqrt(covariance)
        
    # Draw the Ellipse
    for nsig in range(1, 4):
        ax.add_patch(Ellipse(position, nsig * width, nsig * height, 
                            angle, **kwargs))
        
def plot_gmm(gmm, X, label=True, ax=None):
    
    fig, ax = plt.subplots(figsize=(9,7))      
    ax = ax or plt.gca()
    labels = gmm.fit(X).predict(X)
    
    if label:
        ax.scatter(X[:, 0], X[:, 1], c=labels, s=dot_size, cmap=cmap, zorder=2)
    else:
        ax.scatter(X[:, 0], X[:, 1], s=dot_size, zorder=2)
    ax.axis('equal')
    
    w_factor = 0.2 / gmm.weights_.max()
    for pos, covar, w in zip(gmm.means_, gmm.covariances_, gmm.weights_):
        draw_ellipse(pos, covar, ax=ax, alpha=w * w_factor)

gmm = mix.GaussianMixture(n_components=k, random_state=random_state)
plot_gmm(gmm, X3)

# lets test on the stretched data set

gmm = mix.GaussianMixture(n_components=k, random_state=random_state+1)
plot_gmm(gmm, X3_stretched)

gmm2 = mix.GaussianMixture(n_components=2, covariance_type='full',
                          random_state=random_state)
plot_gmm(gmm2, X_mn)

gmm16 = mix.GaussianMixture(n_components=16, covariance_type='full', 
                           random_state=random_state)

plot_gmm(gmm16, X_mn, label=False)
plt.title('Collective Gaussian clusters',
            fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'Collective Gaussian clusters')

Xnew, ynew = gmm16.sample(500)

fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(Xnew[:, 0], Xnew[:, 1]);
ax.set_title('New samples drawn from fitted model',
            fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'New samples drawn from fitted model')

n_components = np.arange(1, 21)
models = [mix.GaussianMixture(n, covariance_type='full',
                             random_state=random_state).fit(X_mn)
         for n in n_components]

fig, ax = plt.subplots(figsize=(9,7))
ax.plot(n_components, [m.bic(X_mn) for m in models], label='BIC')
ax.plot(n_components, [m.aic(X_mn) for m in models], label='AIC')
ax.axvline(np.argmin([m.bic(X_mn) for m in models]), color='blue')
ax.axvline(np.argmin([m.aic(X_mn) for m in models]), color='green')

plt.legend(loc='best')
plt.xlabel('n_components')

Text(0.5, 0, 'n_components')

import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.utils import shuffle
from PIL import Image

# First we read and flatten the image. 
original_img = np.array(Image.open('.\images\sunflower.jpg'), dtype=np.float64) / 255
print(original_img.shape)
original_dimensions = tuple(original_img.shape)
width, height, depth = tuple(original_img.shape)

(1440, 2560, 3)

<>:8: SyntaxWarning: invalid escape sequence '\i'
<>:8: SyntaxWarning: invalid escape sequence '\i'
C:\Users\colin\AppData\Local\Temp\ipykernel_3356\2770556459.py:8: SyntaxWarning: invalid escape sequence '\i'
  original_img = np.array(Image.open('.\images\sunflower.jpg'), dtype=np.float64) / 255

image_flattened = np.reshape(original_img, (width * height, depth))

image_array_sample = shuffle(image_flattened, random_state=0)[:1000]
estimator = KMeans(n_clusters=100, random_state=0)
estimator.fit(image_array_sample)

c:\Users\colin\miniconda3\envs\MachLe\Lib\site-packages\sklearn\cluster\_kmeans.py:1412: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
  super()._check_params_vs_input(X, default_n_init=10)

KMeans(n_clusters=100, random_state=0)

KMeans(n_clusters=100, random_state=0)

cluster_assignments = estimator.predict(image_flattened)

compressed_palette = estimator.cluster_centers_
compressed_img = np.zeros((width, height, compressed_palette.shape[1]))
label_idx = 0
for i in range(width):
    for j in range(height):
        compressed_img[i][j] = compressed_palette[cluster_assignments[label_idx]]
        label_idx += 1

plt.figure(figsize=(12,12))
#plt.subplot(121)
plt.title('Original Image', fontsize=24)
plt.imshow(original_img, origin='lower')
plt.axis('off')
#plt.subplot(122)
plt.figure(figsize=(12,12))
plt.title('Compressed Image', fontsize=24)
plt.imshow(compressed_img,origin='lower')
plt.axis('off')
plt.show()

%matplotlib inline
from IPython.display import set_matplotlib_formats, display
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from cycler import cycler
plt.rcParams['image.cmap'] = "gray"

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()

import matplotlib.pyplot as plt
from sklearn.datasets import fetch_lfw_people
people = fetch_lfw_people(min_faces_per_person=20, resize=2)
image_shape = people.images[0].shape

fig, axes = plt.subplots(2, 5, figsize=(15, 8),
                         subplot_kw={'xticks': (), 'yticks': ()})
for target, image, ax in zip(people.target, people.images, axes.ravel()):
    ax.imshow(image)
    ax.set_title(people.target_names[target])

np.shape(people.images)

(3023, 250, 188)

left = 5
top = 5
right = image_shape[1]-left
bottom = image_shape[0]-top

import PIL

for img in people.images[0:3,:,:]: 
    #pil_img = PIL.Image.fromarray(np.uint8(img*255))
    pil_img = PIL.Image.fromarray(img)
    plt.figure()
    plt.imshow(np.array(pil_img))
    pil_img = pil_img.crop((left, top, right, bottom))
    plt.figure()
    plt.imshow(np.array(pil_img))
    
np.shape(np.array(pil_img))

(240, 178)

print("people.images.shape: {}".format(people.images.shape))
print("Number of classes: {}".format(len(people.target_names)))

people.images.shape: (3023, 250, 188)
Number of classes: 62

people.target_names

array(['Alejandro Toledo', 'Alvaro Uribe', 'Amelie Mauresmo',
       'Andre Agassi', 'Angelina Jolie', 'Ariel Sharon',
       'Arnold Schwarzenegger', 'Atal Bihari Vajpayee', 'Bill Clinton',
       'Carlos Menem', 'Colin Powell', 'David Beckham', 'Donald Rumsfeld',
       'George Robertson', 'George W Bush', 'Gerhard Schroeder',
       'Gloria Macapagal Arroyo', 'Gray Davis', 'Guillermo Coria',
       'Hamid Karzai', 'Hans Blix', 'Hugo Chavez', 'Igor Ivanov',
       'Jack Straw', 'Jacques Chirac', 'Jean Chretien',
       'Jennifer Aniston', 'Jennifer Capriati', 'Jennifer Lopez',
       'Jeremy Greenstock', 'Jiang Zemin', 'John Ashcroft',
       'John Negroponte', 'Jose Maria Aznar', 'Juan Carlos Ferrero',
       'Junichiro Koizumi', 'Kofi Annan', 'Laura Bush',
       'Lindsay Davenport', 'Lleyton Hewitt', 'Luiz Inacio Lula da Silva',
       'Mahmoud Abbas', 'Megawati Sukarnoputri', 'Michael Bloomberg',
       'Naomi Watts', 'Nestor Kirchner', 'Paul Bremer', 'Pete Sampras',
       'Recep Tayyip Erdogan', 'Ricardo Lagos', 'Roh Moo-hyun',
       'Rudolph Giuliani', 'Saddam Hussein', 'Serena Williams',
       'Silvio Berlusconi', 'Tiger Woods', 'Tom Daschle', 'Tom Ridge',
       'Tony Blair', 'Vicente Fox', 'Vladimir Putin', 'Winona Ryder'],
      dtype='<U25')

# count how often each target appears
counts = np.bincount(people.target)
# print counts next to target names:
for i, (count, name) in enumerate(zip(counts, people.target_names)):
    print("{0:25} {1:3}".format(name, count), end='   ')
    if (i + 1) % 3 == 0:
        print()

Alejandro Toledo           39   Alvaro Uribe               35   Amelie Mauresmo            21   
Andre Agassi               36   Angelina Jolie             20   Ariel Sharon               77   
Arnold Schwarzenegger      42   Atal Bihari Vajpayee       24   Bill Clinton               29   
Carlos Menem               21   Colin Powell              236   David Beckham              31   
Donald Rumsfeld           121   George Robertson           22   George W Bush             530   
Gerhard Schroeder         109   Gloria Macapagal Arroyo    44   Gray Davis                 26   
Guillermo Coria            30   Hamid Karzai               22   Hans Blix                  39   
Hugo Chavez                71   Igor Ivanov                20   Jack Straw                 28   
Jacques Chirac             52   Jean Chretien              55   Jennifer Aniston           21   
Jennifer Capriati          42   Jennifer Lopez             21   Jeremy Greenstock          24   
Jiang Zemin                20   John Ashcroft              53   John Negroponte            31   
Jose Maria Aznar           23   Juan Carlos Ferrero        28   Junichiro Koizumi          60   
Kofi Annan                 32   Laura Bush                 41   Lindsay Davenport          22   
Lleyton Hewitt             41   Luiz Inacio Lula da Silva  48   Mahmoud Abbas              29   
Megawati Sukarnoputri      33   Michael Bloomberg          20   Naomi Watts                22   
Nestor Kirchner            37   Paul Bremer                20   Pete Sampras               22   
Recep Tayyip Erdogan       30   Ricardo Lagos              27   Roh Moo-hyun               32   
Rudolph Giuliani           26   Saddam Hussein             23   Serena Williams            52   
Silvio Berlusconi          33   Tiger Woods                23   Tom Daschle                25   
Tom Ridge                  33   Tony Blair                144   Vicente Fox                32   
Vladimir Putin             49   Winona Ryder               24

87*65

5655

mask = np.zeros(people.target.shape, dtype=bool)
for target in np.unique(people.target):
    mask[np.where(people.target == target)[0][:50]] = 1
    
X_people = people.data[mask]
y_people = people.target[mask]

# scale the grey-scale values to be between 0 and 1
# instead of 0 and 255 for better numeric stability:
X_people = X_people / 255.

NumberOfPeople=np.unique(people.target).shape[0]
TargetNames  = [];
n=5

#find the first 5 images from each person
fig, axes = plt.subplots(12, 5, figsize=(15, 30),
                         subplot_kw={'xticks': (), 'yticks': ()})

for target,ax in zip(np.unique(people.target),axes.ravel()):
    #get the first n pictures from each person
    indices=np.where(people.target == target)[0][1:n+1]
    TargetNames.append(people.target_names[target])
    
    image=people.images[indices[0]]
    ax.imshow(image)
    ax.set_title(str(target)+': '+TargetNames[target])

NumberOfPeople

62

#extract eigenfaces from lfw data and transform data
from sklearn.decomposition import PCA
pca = PCA(n_components=100, whiten=True, random_state=0)
X_pca = pca.fit_transform(X_people)
#X_pca = pca.transform(X_people)

image_shape = people.images[0].shape
NumberOfSamples=X_pca.shape[0]

fig, axes = plt.subplots(2, 5, figsize=(15, 8),
                         subplot_kw={'xticks': (), 'yticks': ()})

for ix, target, ax in zip(np.arange(NumberOfSamples), y_people, axes.ravel()):
    image=np.reshape(pca.inverse_transform(X_pca[ix,:]),image_shape)
    ax.imshow(image)
    ax.set_title(str(y_people[ix])+': '+people.target_names[target])

# apply DBSCAN with default parameters
from sklearn.cluster import DBSCAN
dbscan = DBSCAN()
labels = dbscan.fit_predict(X_pca)
print("Unique labels: {}".format(np.unique(labels)))

Unique labels: [-1]

for eps in np.linspace(6,8,51):
    print("\neps={}".format(eps))
    dbscan = DBSCAN(eps=eps, min_samples=3)
    labels = dbscan.fit_predict(X_pca)
    print("Number of clusters: {}".format(len(np.unique(labels))))
    print("Cluster sizes: {}".format(np.bincount(labels + 1)))

eps=6.0
Number of clusters: 3
Cluster sizes: [2052    7    4]

eps=6.04
Number of clusters: 3
Cluster sizes: [2052    7    4]

eps=6.08
Number of clusters: 3
Cluster sizes: [2052    7    4]

eps=6.12
Number of clusters: 3
Cluster sizes: [2052    7    4]

eps=6.16
Number of clusters: 3
Cluster sizes: [2051    7    5]

eps=6.2
Number of clusters: 3
Cluster sizes: [2051    7    5]

eps=6.24
Number of clusters: 3
Cluster sizes: [2051    7    5]

eps=6.28
Number of clusters: 3
Cluster sizes: [2051    7    5]

eps=6.32
Number of clusters: 3
Cluster sizes: [2051    7    5]

eps=6.36
Number of clusters: 3
Cluster sizes: [2051    7    5]

eps=6.4
Number of clusters: 3
Cluster sizes: [2051    7    5]

eps=6.44
Number of clusters: 2
Cluster sizes: [2051   12]

eps=6.48
Number of clusters: 3
Cluster sizes: [2048    3   12]

eps=6.52
Number of clusters: 3
Cluster sizes: [2048    3   12]

eps=6.5600000000000005
Number of clusters: 3
Cluster sizes: [2048    3   12]

eps=6.6
Number of clusters: 3
Cluster sizes: [2048    3   12]

eps=6.64
Number of clusters: 3
Cluster sizes: [2048    3   12]

eps=6.68
Number of clusters: 3
Cluster sizes: [2048    3   12]

eps=6.72
Number of clusters: 4
Cluster sizes: [2043    3   13    4]

eps=6.76
Number of clusters: 5
Cluster sizes: [2040    3   13    3    4]

eps=6.8
Number of clusters: 8
Cluster sizes: [2030    3   13    3    4    3    4    3]

eps=6.84
Number of clusters: 11
Cluster sizes: [2020    3   13    3    4    3    3    4    3    4    3]

eps=6.88
Number of clusters: 12
Cluster sizes: [2016    3   13    3    4    3    4    4    3    4    3    3]

eps=6.92
Number of clusters: 14
Cluster sizes: [2007    3   14    6    3    3    3    4    4    3    3    4    3    3]

eps=6.96
Number of clusters: 14
Cluster sizes: [2005    3   14    7    4    3    3    4    4    3    3    4    3    3]

eps=7.0
Number of clusters: 14
Cluster sizes: [2004    3   14    7    4    3    3    4    4    3    3    5    3    3]

eps=7.04
Number of clusters: 12
Cluster sizes: [2003    4   14   13    4    3    3    4    4    5    3    3]

eps=7.08
Number of clusters: 12
Cluster sizes: [2002    4    3   14   13    4    3    5    4    5    3    3]

eps=7.12
Number of clusters: 14
Cluster sizes: [1996    4    3   14   13    3    4    3    5    3    4    3    5    3]

eps=7.16
Number of clusters: 12
Cluster sizes: [1991    4    3   14   23    8    3    5    3    3    3    3]

eps=7.2
Number of clusters: 13
Cluster sizes: [1979   13    3    4   14   26    3    5    3    4    3    3    3]

eps=7.24
Number of clusters: 12
Cluster sizes: [1973   16    3    5   42    3    5    3    4    3    3    3]

eps=7.28
Number of clusters: 12
Cluster sizes: [1965   63    3    5    3    5    3    4    3    3    3    3]

eps=7.32
Number of clusters: 12
Cluster sizes: [1959   67    3    5    3    6    3    4    3    4    3    3]

eps=7.36
Number of clusters: 13
Cluster sizes: [1950   71    4    5    4    3    6    3    4    3    4    3    3]

eps=7.4
Number of clusters: 13
Cluster sizes: [1941   77    4    7    4    6    3    4    4    3    4    3    3]

eps=7.4399999999999995
Number of clusters: 13
Cluster sizes: [1937   80    7    4    7    4    3    4    4    3    4    3    3]

eps=7.48
Number of clusters: 14
Cluster sizes: [1926   90    7    5    4    3    3    4    4    3    5    3    3    3]

eps=7.52
Number of clusters: 14
Cluster sizes: [1917   93   12    5    3    5    3    3    4    5    3    4    3    3]

eps=7.5600000000000005
Number of clusters: 14
Cluster sizes: [1911   97   12    6    3    5    3    3    4    5    4    4    3    3]

eps=7.6
Number of clusters: 14
Cluster sizes: [1901  109   12    3    5    3    5    3    3    4    5    4    3    3]

eps=7.640000000000001
Number of clusters: 16
Cluster sizes: [1872  142    4    4    3    4    3    3    5    3    3    3    3    5
    3    3]

eps=7.68
Number of clusters: 15
Cluster sizes: [1866  148    4    4    3    8    4    3    3    3    3    3    5    3
    3]

eps=7.72
Number of clusters: 12
Cluster sizes: [1852  169    4    4   10    4    3    3    3    3    3    5]

eps=7.76
Number of clusters: 8
Cluster sizes: [1841  200    4    4    3    3    3    5]

eps=7.8
Number of clusters: 8
Cluster sizes: [1824  217    4    4    3    3    5    3]

eps=7.84
Number of clusters: 8
Cluster sizes: [1815  226    4    4    3    3    5    3]

eps=7.88
Number of clusters: 8
Cluster sizes: [1805  236    4    4    3    3    5    3]

eps=7.92
Number of clusters: 6
Cluster sizes: [1797  250    4    4    5    3]

eps=7.96
Number of clusters: 5
Cluster sizes: [1788  263    4    5    3]

eps=8.0
Number of clusters: 6
Cluster sizes: [1762  287    3    3    5    3]

dbscan = DBSCAN(min_samples=3, eps=7.640000000000001)
labels = dbscan.fit_predict(X_pca)

for cluster in range(max(labels) + 1):
    mask = labels == cluster
    n_images =  np.sum(mask)
    if n_images<7:
        fig, axes = plt.subplots(1, n_images, figsize=(n_images * 1.5, 4),
                             subplot_kw={'xticks': (), 'yticks': ()})
        for image, label, ax in zip(X_people[mask], y_people[mask], axes):

            ax.imshow(image.reshape(image_shape), vmin=0, vmax=1)
            ax.set_title(people.target_names[label].split()[-1])

# %% using other cluster algorithms learner on the pca transformed data
from time import time
from sklearn import cluster
from sklearn.neighbors import kneighbors_graph

n_clusters=14

clustering_names = ['SpectralClustering', 'Ward', 'AverageLinkage']

connectivity = kneighbors_graph(X_pca, n_neighbors=n_clusters, include_self=False)
# make connectivity symmetric
connectivity = 0.5 * (connectivity + connectivity.T)

spectral = cluster.SpectralClustering(n_clusters=n_clusters,
                                          eigen_solver='arpack',
                                          affinity="nearest_neighbors")

ward = cluster.AgglomerativeClustering(n_clusters=n_clusters, linkage='ward',
                                           connectivity=connectivity)

average_linkage = cluster.AgglomerativeClustering(
        linkage="average", affinity="cityblock", n_clusters=n_clusters,
        connectivity=connectivity)


clustering_algorithms = [spectral, ward, average_linkage]

# %matplotlib inline
for name, algorithm in zip(clustering_names, clustering_algorithms):
    # predict cluster memberships
    print(algorithm)
    t0 = time()
    algorithm.fit(X_pca)
    t1 = time()
        
    if hasattr(algorithm, 'labels_'):
        labels = algorithm.labels_.astype(np.int)
    else:
        labels = algorithm.predict(X_pca)
    
    print("%s: %.2g sec" % (name,t1 - t0))
    print('labels found: %i' % (max(labels) + 1))
    print("_____________________________________________")
    print("       %s                                     " % (name))
    print("_____________________________________________")
    
    for cluster in range(max(labels) + 1):
        mask = labels == cluster
        ind=np.where(mask==True)[0]
        n_images = np.size(ind)
        submask=np.zeros(X_pca.shape[0])
        submask=submask.astype(dtype=bool)
        submask[ind]=True
        #n_images =  np.sum(mask)
        #print(n_images)
            
        max_image=np.min([n_images,8])
        print('max image: %i\n' % (max_image))
        fig, axes = plt.subplots(1, max_image, figsize=(max_image * 3, 3),
                         subplot_kw={'xticks': (), 'yticks': ()})
        
        if max_image==1:
            print(ind[0])
            image=X_people[ind[0]]
            label=y_people[ind[0]]
            plt.imshow(image.reshape(image_shape), vmin=0, vmax=1)
            plt.title(people.target_names[label].split()[-1])
        else:
            for image, label, ax in zip(X_people[submask], y_people[submask], axes):
                ax.imshow(image.reshape(image_shape), vmin=0, vmax=1)
                ax.set_title(people.target_names[label].split()[-1])
        
        plt.show()

SpectralClustering(affinity='nearest_neighbors', eigen_solver='arpack',
                   n_clusters=14)

C:\Users\christoph.wuersch\AppData\Local\Temp\ipykernel_22540\2706636364.py:37: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
  labels = algorithm.labels_.astype(np.int)

SpectralClustering: 1.7 sec
labels found: 14
_____________________________________________
       SpectralClustering                                     
_____________________________________________
max image: 8

max image: 8

max image: 8

max image: 8

max image: 8

max image: 8

max image: 8

max image: 8

import warnings
warnings.filterwarnings("ignore") 
from IPython.core.display import display, HTML

import time

import pandas as pd
#import pandas_datareader.data as web
import numpy as np
import scipy.stats as scs
from scipy.stats import multivariate_normal as mvn
import sklearn.mixture as mix

import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

# import class heights
f = 'https://raw.githubusercontent.com/BlackArbsCEO/Mixture_Models/K-Means%2C-E-M%2C-Mixture-Models/Class_heights.csv'

data = pd.read_csv(f)
data.to_csv('Class_heights.csv')
# data.info()

height = data['Height (in)']
data.head()

data.describe()

# Code sourced from:
#    http://people.duke.edu/~ccc14/sta-663/EMAlgorithm.html

def em_gmm_orig(xs, pis, mus, sigmas, tol=0.01, max_iter=20):

    n, p = xs.shape
    k = len(pis)

    ll_old = 0
    for itercount in range(max_iter):
        print('\nIteration: ', itercount)
        print()
        exp_A = []
        exp_B = []
        ll_new = 0

        # E-step
        ws = np.zeros((k, n))
        for j in range(len(mus)):
            for i in range(n):
                ws[j, i] = pis[j] * mvn(mus[j], sigmas[j]).pdf(xs[i])
        ws /= ws.sum(0)

        # M-step
        pis = np.zeros(k)
        for j in range(len(mus)):
            for i in range(n):
                pis[j] += ws[j, i]
        pis /= n

        mus = np.zeros((k, p))
        for j in range(k):
            for i in range(n):
                mus[j] += ws[j, i] * xs[i]
            mus[j] /= ws[j, :].sum()
        
        sigmas = np.zeros((k, p, p))
        for j in range(k):
            for i in range(n):
                ys = np.reshape(xs[i]- mus[j], (2,1))
                sigmas[j] += ws[j, i] * np.dot(ys, ys.T)
            sigmas[j] /= ws[j,:].sum()

        new_mus = (np.diag(mus)[0], np.diag(mus)[1])
        new_sigs = (np.unique(np.diag(sigmas[0]))[0], np.unique(np.diag(sigmas[1]))[0])
        df = (pd.DataFrame(index=[1, 2]).assign(mus = new_mus).assign(sigs = new_sigs))
        
        xx = np.linspace(0, 100, 100)
        yy = scs.multivariate_normal.pdf(xx, mean=new_mus[0], cov=new_sigs[0])
        
        colors = sns.color_palette('Dark2', 3)
        fig, ax = plt.subplots(figsize=(9, 7))
        ax.set_ylim(-0.001, np.max(yy))
        ax.plot(xx, yy, color=colors[1])
        ax.axvline(new_mus[0], ymin=0., color=colors[1])
        ax.fill_between(xx, 0, yy, alpha=0.5, color=colors[1])
        lo, hi = ax.get_ylim()
        ax.annotate(f'$\mu_1$: {new_mus[0]:3.2f}', 
                    fontsize=12, fontweight='demi',
                    xy=(new_mus[0], (hi-lo) / 2), 
                    xycoords='data', xytext=(80, (hi-lo) / 2),
                    arrowprops=dict(facecolor='black', connectionstyle="arc3,rad=0.2",shrink=0.05))
        ax.fill_between(xx, 0, yy, alpha=0.5, color=colors[2])
        
        yy2 = scs.multivariate_normal.pdf(xx, mean=new_mus[1], cov=new_sigs[1])
        
        ax.plot(xx, yy2, color=colors[2])
        ax.axvline(new_mus[1], ymin=0., color=colors[2])
        lo, hi = ax.get_ylim()
        ax.annotate(f'$\mu_2$: {new_mus[1]:3.2f}', 
                    fontsize=12, fontweight='demi',
            xy=(new_mus[1], (hi-lo) / 2), xycoords='data', xytext=(25, (hi-lo) / 2),
            arrowprops=dict(facecolor='black', connectionstyle="arc3,rad=0.2",shrink=0.05))
        ax.fill_between(xx, 0, yy2, alpha=0.5, color=colors[2])
        
        dot_kwds = dict(markerfacecolor='white', markeredgecolor='black', markeredgewidth=1, markersize=10)
        ax.plot(height, len(height)*[0], 'o', **dot_kwds)
        ax.set_ylim(-0.001, np.max(yy2))
        
        figureFileName="EM_GMM_iter_%i.png" % itercount
        print(figureFileName)
        plt.savefig(figureFileName,dpi=600)
        
        
        print(df.T)   
        
        # update complete log likelihoood
        ll_new = 0.0
        for i in range(n):
            s = 0
            for j in range(k):
                s += pis[j] * mvn(mus[j], sigmas[j]).pdf(xs[i])
            ll_new += np.log(s)
        print(f'log_likelihood: {ll_new:3.4f}')
        if np.abs(ll_new - ll_old) < tol:
            break
        ll_old = ll_new
        
    return ll_new, pis, mus, sigmas

height = data['Height (in)']
n = len(height)

# Ground truthish
_mus = np.array([[0, data.groupby('Gender').mean().iat[0, 0]], 
                 [data.groupby('Gender').mean().iat[1, 0], 0]])
_sigmas = np.array([[[5, 0], [0, 5]], 
                    [[5, 0],[0, 5]]])
_pis = np.array([0.5, 0.5]) # priors

# initial random guesses for parameters
np.random.seed(0)

pis = np.random.random(2)
pis /= pis.sum()
mus = np.random.random((2,2))
sigmas = np.array([np.eye(2)] * 2) * height.std()

# generate our noisy x values
xs = np.concatenate([np.random.multivariate_normal(mu, sigma, int(pi*n))
                    for pi, mu, sigma in zip(_pis, _mus, _sigmas)])

ll, pis, mus, sigmas = em_gmm_orig(xs, pis, mus, sigmas)

# In the below plots the white dots represent the observed heights.

Iteration:  0

EM_GMM_iter_0.png
               1           2
mus    61.362928   59.659685
sigs  469.240750  244.382352
log_likelihood: -141.8092

Iteration:  1

EM_GMM_iter_1.png
              1          2
mus    68.73773  63.620554
sigs  109.85442   7.228183
log_likelihood: -118.0520

Iteration:  2

EM_GMM_iter_2.png
              1          2
mus   70.569842  63.688825
sigs   4.424452   3.139277
log_likelihood: -100.2591

Iteration:  3

EM_GMM_iter_3.png
              1          2
mus   70.569842  63.688825
sigs   4.424427   3.139278
log_likelihood: -100.2591

# Let's define some demo variables and make some blobs

# demo variables

k = 4
n_draws = 500
sigma = .7
random_state = 0
dot_size = 50
cmap = 'viridis'

# make blobs

from sklearn.datasets import make_blobs

X, y_true = make_blobs(n_samples = n_draws,
                       centers = k,
                       cluster_std = sigma,
                       random_state = random_state)

fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(X[:, 0], X[:, 1], s=dot_size)
plt.title('k-means make blobs', fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'k-means make blobs')

# sample implementation
# code sourced from: 
#   http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/05.11-K-Means.ipynb

from sklearn.metrics import pairwise_distances_argmin

def find_clusters(X, n_clusters, rseed=2):
    # 1. Random initialization (choose random clusters)
    rng = np.random.RandomState(rseed)
    i = rng.permutation(X.shape[0])[:n_clusters]
    centers = X[i]

    while True:
        # 2a. Assign labels based on closest center
        labels = pairwise_distances_argmin(X, centers)

        # 2b. Find new centers from means of points
        new_centers = np.array([X[labels == i].mean(0) 
                                for i in range(n_clusters)])

        # 2c. Check for convergence
        if np.all(centers == new_centers):
            break
        centers = new_centers
    
    return centers, labels

# now let's compare this to the sklearn's KMeans() algorithm

# fit k-means to blobs

from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=k)
kmeans.fit(X)
y_kmeans = kmeans.predict(X)

# visualize prediction
fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(X[:, 0], X[:, 1], c=y_kmeans, s=dot_size, cmap=cmap)

# get centers for plot
centers = kmeans.cluster_centers_
ax.scatter(centers[:, 0], centers[:, 1], c='black', s=200, alpha=0.75)
plt.title('sklearn k-means', fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'sklearn k-means')

# let's test the implementation

centers, labels = find_clusters(X, k)

fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(X[:, 0], X[:, 1], c=labels, s=dot_size, cmap=cmap)
plt.title('find_clusters() k-means func', fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'find_clusters() k-means func')

# code sourced from:
#   http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/06.00-Figure-Code.ipynb#Covariance-Type

from ipywidgets import interact

def plot_kmeans_interactive(min_clusters=1, max_clusters=6):
    X, y = make_blobs(n_samples=300, centers=4,
                      random_state=0, cluster_std=0.60)
        
    def plot_points(X, labels, n_clusters):
        plt.scatter(X[:, 0], X[:, 1], c=labels, s=50, cmap='viridis',
                    vmin=0, vmax=n_clusters - 1);
            
    def plot_centers(centers):
        plt.scatter(centers[:, 0], centers[:, 1], marker='o',
                    c=np.arange(centers.shape[0]),
                    s=200, cmap='viridis')
        plt.scatter(centers[:, 0], centers[:, 1], marker='o',
                    c='black', s=50)
            

    def _kmeans_step(frame=0, n_clusters=4):
        rng = np.random.RandomState(2)
        labels = np.zeros(X.shape[0])
        centers = rng.randn(n_clusters, 2)

        nsteps = frame // 3

        for i in range(nsteps + 1):
            old_centers = centers
            if i < nsteps or frame % 3 > 0:
                labels = pairwise_distances_argmin(X, centers)

            if i < nsteps or frame % 3 > 1:
                centers = np.array([X[labels == j].mean(0)
                                    for j in range(n_clusters)])
                nans = np.isnan(centers)
                centers[nans] = old_centers[nans]

        # plot the data and cluster centers
        plot_points(X, labels, n_clusters)
        plot_centers(old_centers)

        # plot new centers if third frame
        if frame % 3 == 2:
            for i in range(n_clusters):
                plt.annotate('', centers[i], old_centers[i], 
                             arrowprops=dict(arrowstyle='->', linewidth=1))
            plot_centers(centers)

        plt.xlim(-4, 4)
        plt.ylim(-2, 10)

        if frame % 3 == 1:
            plt.text(3.8, 9.5, "1. Reassign points to nearest centroid",
                     ha='right', va='top', size=14)
        elif frame % 3 == 2:
            plt.text(3.8, 9.5, "2. Update centroids to cluster means",
                     ha='right', va='top', size=14)
    
    return interact(_kmeans_step, frame=[0, 10, 20, 30, 40, 50, 300],
                    n_clusters=np.arange(min_clusters, max_clusters+1))

plot_kmeans_interactive()

<function __main__.plot_kmeans_interactive.<locals>._kmeans_step(frame=0, n_clusters=4)>

centers, labels = find_clusters(X, k, rseed=11)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('sub-optimal clustering', fontsize=18, fontweight='demi')
ax.scatter(X[:, 0], X[:, 1], c=labels, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x13d3c1efd60>

from sklearn.cluster import KMeans

labels6 = KMeans(6, random_state=random_state).fit_predict(X)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('too many clusters', fontsize=18, fontweight='demi')
ax.scatter(X[:, 0], X[:, 1], c=labels6, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x13d3c876ca0>

from sklearn.datasets import make_moons

X_mn, y_mn = make_moons(500, noise=.07, random_state=random_state)

labelsM = KMeans(2, random_state=random_state).fit_predict(X_mn)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('linear separation not possible', fontsize=18, fontweight='demi')
ax.scatter(X_mn[:, 0], X_mn[:, 1], c=labelsM, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x13d3ca918b0>

from sklearn.cluster import SpectralClustering

model = SpectralClustering(n_clusters=2, affinity='nearest_neighbors',
                          assign_labels='kmeans')

labelsS = model.fit_predict(X_mn)

fig, ax = plt.subplots(figsize=(9,7))
ax.set_title('kernal transform to higher dimension\nlinear separation is possible', fontsize=18, fontweight='demi')
plt.scatter(X_mn[:, 0], X_mn[:, 1], c=labelsS, s=dot_size, cmap=cmap)

<matplotlib.collections.PathCollection at 0x13d3bee7e80>

# k-means weaknesses that mixture models address directly
# code sourced from:
#   http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/05.12-Gaussian-Mixtures.ipynb

from scipy.spatial.distance import cdist

def plot_kmeans(kmeans, X, n_clusters=k, rseed=2, ax=None):
    labels = kmeans.fit_predict(X)
    
    # plot input data
    #ax = ax or plt.gca() # <-- nice trick
    fig, ax = plt.subplots(figsize=(9,7))    
    ax.axis('equal')
    ax.scatter(X[:, 0], X[:, 1],
               c=labels, s=dot_size, cmap=cmap, zorder=2)
    
    # plot the representation of Kmeans model
    centers = kmeans.cluster_centers_
    radii = [cdist(X[labels==i], [center]).max() 
             for i, center in enumerate(centers)]
    
    for c, r in zip(centers, radii):
        ax.add_patch(plt.Circle(c, r, fc='#CCCCCC',edgecolor='slategrey',
                                lw=4, alpha=0.5, zorder=1))
    return      
                     
X3, y_true = make_blobs(n_samples = 400,
                       centers = k,
                       cluster_std = .6,
                       random_state = random_state)
X3 = X3[:, ::-1] # better plotting

kmeans = KMeans(n_clusters=k, random_state=random_state)
plot_kmeans(kmeans, X3)
plt.title('Clusters are hard circular boundaries', fontsize=18, fontweight='demi')
plt.savefig('Kmeans_circular.png',dpi=600)

rng = np.random.RandomState(13)
X3_stretched = np.dot(X3, rng.randn(2, 2))

kmeans = KMeans(n_clusters=k, random_state=random_state)
plot_kmeans(kmeans, X3_stretched)
plt.title('Clusters cannot adjust to elliptical data structures',
         fontsize=18, fontweight='demi')
plt.savefig('Kmeans_elliptical.png',dpi=600)

# code sourced from:
#  http://nbviewer.jupyter.org/github/jakevdp/PythonDataScienceHandbook/blob/master/notebooks/05.12-Gaussian-Mixtures.ipynb
from matplotlib.patches import Ellipse 

def draw_ellipse(position, covariance, ax=None, **kwargs):
    """Draw an ellipse with a given position and covariance"""
    
    # Convert covariance to principal axes
    if covariance.shape == (2, 2):
        U, s, Vt = np.linalg.svd(covariance)
        angle = np.degrees(np.arctan2(U[1, 0], U[0, 0]))
        width, height = 2 * np.sqrt(s)
    else:
        angle = 0
        width, height = 2 * np.sqrt(covariance)
        
    # Draw the Ellipse
    for nsig in range(1, 4):
        ax.add_patch(Ellipse(position, nsig * width, nsig * height, 
                            angle, **kwargs))
        
def plot_gmm(gmm, X, label=True, ax=None):
    
    fig, ax = plt.subplots(figsize=(9,7))      
    ax = ax or plt.gca()
    labels = gmm.fit(X).predict(X)
    
    if label:
        ax.scatter(X[:, 0], X[:, 1], c=labels, s=dot_size, cmap=cmap, zorder=2)
    else:
        ax.scatter(X[:, 0], X[:, 1], s=dot_size, zorder=2)
    ax.axis('equal')
    
    w_factor = 0.2 / gmm.weights_.max()
    for pos, covar, w in zip(gmm.means_, gmm.covariances_, gmm.weights_):
        draw_ellipse(pos, covar, ax=ax, alpha=w * w_factor)

gmm = mix.GaussianMixture(n_components=k, random_state=random_state)
plot_gmm(gmm, X3)
plt.title('GMM isotropic clusters', fontsize=18, fontweight='demi')
plt.savefig('GMM_circular.png',dpi=600)

# lets test on the stretched data set

gmm = mix.GaussianMixture(n_components=k, random_state=random_state+1)
plot_gmm(gmm, X3_stretched)
plt.title('GMM elliptical clusters', fontsize=18, fontweight='demi')
plt.savefig('GMM_elliptical.png',dpi=600)

gmm2 = mix.GaussianMixture(n_components=2, covariance_type='full',
                          random_state=random_state)
plot_gmm(gmm2, X_mn)

gmm16 = mix.GaussianMixture(n_components=16, covariance_type='full', 
                           random_state=random_state)

plot_gmm(gmm16, X_mn, label=False)
plt.title('Collective Gaussian clusters',
            fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'Collective Gaussian clusters')

Xnew, ynew = gmm16.sample(500)

fig, ax = plt.subplots(figsize=(9,7))
ax.scatter(Xnew[:, 0], Xnew[:, 1]);
ax.set_title('New samples drawn from fitted model',
            fontsize=18, fontweight='demi')

Text(0.5, 1.0, 'New samples drawn from fitted model')

n_components = np.arange(1, 21)
models = [mix.GaussianMixture(n, covariance_type='full',
                             random_state=random_state).fit(X_mn)
         for n in n_components]

fig, ax = plt.subplots(figsize=(9,7))
ax.plot(n_components, [m.bic(X_mn) for m in models], label='BIC')
ax.plot(n_components, [m.aic(X_mn) for m in models], label='AIC')
ax.axvline(np.argmin([m.bic(X_mn) for m in models]), color='blue')
ax.axvline(np.argmin([m.aic(X_mn) for m in models]), color='green')

plt.legend(loc='best')
plt.xlabel('n_components')

Text(0.5, 0, 'n_components')

	LATITUDE	LONGITUDE	NUMBER OF PERSONS INJURED	NUMBER OF PERSONS KILLED	NUMBER OF PEDESTRIANS INJURED	NUMBER OF PEDESTRIANS KILLED	NUMBER OF CYCLIST INJURED	NUMBER OF CYCLIST KILLED	NUMBER OF MOTORIST INJURED	NUMBER OF MOTORIST KILLED	COLLISION_ID
count	1.796344e+06	1.796344e+06	2.026629e+06	2.026616e+06	2.026647e+06	2.026647e+06	2.026647e+06	2.026647e+06	2.026647e+06	2.026647e+06	2.026647e+06
mean	4.062776e+01	-7.375228e+01	3.036264e-01	1.454148e-03	5.528639e-02	7.263228e-04	2.630749e-02	1.115142e-04	2.187850e-01	5.950716e-04	3.122850e+06
std	1.980800e+00	3.726823e+00	6.948115e-01	4.018541e-02	2.415744e-01	2.743065e-02	1.619910e-01	1.060607e-02	6.559466e-01	2.661214e-02	1.504145e+06
min	0.000000e+00	-2.013600e+02	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	2.200000e+01
25%	4.066792e+01	-7.397492e+01	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	3.142782e+06
50%	4.072097e+01	-7.392732e+01	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	3.649550e+06
75%	4.076956e+01	-7.386668e+01	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	4.156458e+06
max	4.334444e+01	0.000000e+00	4.300000e+01	8.000000e+00	2.700000e+01	6.000000e+00	4.000000e+00	2.000000e+00	4.300000e+01	5.000000e+00	4.663399e+06

BOROUGH	BRONX	BROOKLYN	MANHATTAN	QUEENS	STATEN ISLAND
CONTRIBUTING FACTOR VEHICLE 1
1	NaN	3.0	1.0	2.0	2.0
80	12.0	19.0	9.0	29.0	1.0
Accelerator Defective	133.0	247.0	121.0	217.0	50.0
Aggressive Driving/Road Rage	1271.0	1949.0	1344.0	1396.0	229.0
Alcohol Involvement	2533.0	4737.0	2278.0	4738.0	846.0
Animals Action	128.0	216.0	87.0	279.0	223.0
Backing Unsafely	8822.0	17736.0	11500.0	18473.0	2524.0
Brakes Defective	782.0	1407.0	716.0	1128.0	299.0
Cell Phone (hand-Held)	72.0	97.0	53.0	72.0	18.0
Cell Phone (hand-held)	8.0	21.0	15.0	11.0	1.0

	Outcome	Feature 1	Feature 2	...	Feature d
Observation 1	$y_1$	$x_{11}$	$x_{12}$	...	$x_{1d}$
Observation 2	$y_2$	$x_{21}$	$x_{22}$	...	$x_{2d}$
...	...	...	...	...	...
Observation n	$y_n$	$x_{n1}$	$x_{n2}$	...	$x_{nd}$

	Temp3pm	Humidity3pm	Cloud3pm	Pressure3pm	WindSpeed3pm	Sunshine	Rainfall
count	328.000000	328.000000	328.000000	328.000000	328.000000	328.000000	328.000000
mean	19.556402	44.003049	4.000000	1016.530793	18.185976	8.014939	1.440854
std	6.644311	16.605975	2.652101	6.469774	8.926759	3.506646	4.289427
min	5.100000	13.000000	0.000000	996.800000	4.000000	0.000000	0.000000
25%	14.500000	32.000000	1.000000	1012.400000	11.000000	6.000000	0.000000
50%	18.850000	42.500000	4.000000	1016.900000	17.000000	8.750000	0.000000
75%	24.225000	54.000000	7.000000	1021.125000	24.000000	10.700000	0.200000
max	34.500000	93.000000	8.000000	1033.200000	52.000000	13.600000	39.800000

	Temp3pm	Humidity3pm	Cloud3pm	Pressure3pm	WindSpeed3pm	Sunshine	Rainfall
Temp3pm	1.000000	-0.569348	-0.181667	-0.332099	-0.239119	0.463721	-0.089740
Humidity3pm	-0.569348	1.000000	0.530715	-0.047607	0.015860	-0.760267	0.287244
Cloud3pm	-0.181667	0.530715	1.000000	-0.146235	0.011625	-0.657198	0.134894
Pressure3pm	-0.332099	-0.047607	-0.146235	1.000000	-0.318008	-0.024120	-0.263710
WindSpeed3pm	-0.239119	0.015860	0.011625	-0.318008	1.000000	0.046140	0.058151
Sunshine	0.463721	-0.760267	-0.657198	-0.024120	0.046140	1.000000	-0.158062
Rainfall	-0.089740	0.287244	0.134894	-0.263710	0.058151	-0.158062	1.000000

	CRASH DATE	CRASH TIME	BOROUGH	ZIP CODE	LATITUDE	LONGITUDE	LOCATION	ON STREET NAME	CROSS STREET NAME	OFF STREET NAME	...	CONTRIBUTING FACTOR VEHICLE 2	CONTRIBUTING FACTOR VEHICLE 3	CONTRIBUTING FACTOR VEHICLE 4	CONTRIBUTING FACTOR VEHICLE 5	COLLISION_ID	VEHICLE TYPE CODE 1	VEHICLE TYPE CODE 2	VEHICLE TYPE CODE 3	VEHICLE TYPE CODE 4	VEHICLE TYPE CODE 5
0	09/11/2021	2:39	NaN	NaN	NaN	NaN	NaN	WHITESTONE EXPRESSWAY	20 AVENUE	NaN	...	Unspecified	NaN	NaN	NaN	4455765	Sedan	Sedan	NaN	NaN	NaN
1	03/26/2022	11:45	NaN	NaN	NaN	NaN	NaN	QUEENSBORO BRIDGE UPPER	NaN	NaN	...	NaN	NaN	NaN	NaN	4513547	Sedan	NaN	NaN	NaN	NaN
2	06/29/2022	6:55	NaN	NaN	NaN	NaN	NaN	THROGS NECK BRIDGE	NaN	NaN	...	Unspecified	NaN	NaN	NaN	4541903	Sedan	Pick-up Truck	NaN	NaN	NaN
3	09/11/2021	9:35	BROOKLYN	11208	40.667202	-73.866500	(40.667202, -73.8665)	NaN	NaN	1211 LORING AVENUE	...	NaN	NaN	NaN	NaN	4456314	Sedan	NaN	NaN	NaN	NaN
4	12/14/2021	8:13	BROOKLYN	11233	40.683304	-73.917274	(40.683304, -73.917274)	SARATOGA AVENUE	DECATUR STREET	NaN	...	NaN	NaN	NaN	NaN	4486609	NaN	NaN	NaN	NaN	NaN

	CRASH DATE	BOROUGH	NUMBER OF PERSONS INJURED
0	2021-09-11	NaN	2.0
1	2022-03-26	NaN	1.0
2	2022-06-29	NaN	0.0
3	2021-09-11	BROOKLYN	0.0
4	2021-12-14	BROOKLYN	0.0
...	...	...	...
2026642	2023-07-03	NaN	0.0
2026643	2023-07-22	BRONX	1.0
2026644	2023-07-02	MANHATTAN	0.0
2026645	2023-07-22	QUEENS	1.0
2026646	2023-07-22	QUEENS	0.0

	BOROUGH	NUMBER OF PERSONS INJURED
0	BRONX	66156.0
1	BROOKLYN	144095.0
2	MANHATTAN	64655.0
3	QUEENS	110271.0
4	STATEN ISLAND	16405.0

	BOROUGH	SUM INJURED	MAX INJURED
0	BRONX	66156.0	31.0
1	BROOKLYN	144095.0	43.0
2	MANHATTAN	64655.0	27.0
3	QUEENS	110271.0	34.0
4	STATEN ISLAND	16405.0	22.0

		NUMBER OF PERSONS INJURED	CONTRIBUTING FACTOR VEHICLE 1
BOROUGH	VEHICLE TYPE CODE 1
BRONX	197209	0.0	Unspecified
	1S	0.0	Passing or Lane Usage Improper
	2 DOO	0.0	Unspecified
	2 dr sedan	49.0	UnspecifiedUnspecifiedOther VehicularUnspecifi...
	3-Door	12.0	Driver Inattention/DistractionPassing Too Clos...
...	...	...	...
STATEN ISLAND	trailer	1.0	Driver InexperienceBacking Unsafely
	unkno	0.0	Turning Improperly
	usps	0.0	Failure to Keep Right
	van	1.0	Turning ImproperlyFailure to Yield Right-of-Wa...
	van t	1.0	Unsafe Speed

	x1	x2	y
0	0.00	0.20	0
1	0.40	0.35	0
2	0.15	0.00	0
3	0.05	0.10	0
4	0.40	0.40	0
5	0.20	0.25	0
6	0.00	0.40	0
7	0.45	0.35	0
8	1.00	0.85	1
9	0.85	0.95	1
10	0.90	1.00	1
11	0.70	0.65	1
12	0.65	0.75	1
13	0.95	0.90	1
14	1.00	0.90	1
15	0.80	0.95	1

	Date	Location	MinTemp	MaxTemp	Rainfall	Evaporation	Sunshine	WindGustDir	WindGustSpeed	WindDir9am	...	Humidity3pm	Pressure9am	Pressure3pm	Cloud9am	Cloud3pm	Temp9am	Temp3pm	RainToday	RISK_MM	RainTomorrow
0	2007-11-01	Canberra	8.0	24.3	0.0	3.4	6.3	NW	30.0	SW	...	29	1019.7	1015.0	7	7	14.4	23.6	No	3.6	Yes
1	2007-11-02	Canberra	14.0	26.9	3.6	4.4	9.7	ENE	39.0	E	...	36	1012.4	1008.4	5	3	17.5	25.7	Yes	3.6	Yes
2	2007-11-03	Canberra	13.7	23.4	3.6	5.8	3.3	NW	85.0	N	...	69	1009.5	1007.2	8	7	15.4	20.2	Yes	39.8	Yes
3	2007-11-04	Canberra	13.3	15.5	39.8	7.2	9.1	NW	54.0	WNW	...	56	1005.5	1007.0	2	7	13.5	14.1	Yes	2.8	Yes
4	2007-11-05	Canberra	7.6	16.1	2.8	5.6	10.6	SSE	50.0	SSE	...	49	1018.3	1018.5	7	7	11.1	15.4	Yes	0.0	No

	Coefficients multivariate model
Humidity3pm	-16.236408
Cloud3pm	2.993911
Pressure3pm	-19.235234
WindSpeed3pm	-13.840040
WindDir3pm	-2.875112
Sunshine	4.779577
Rainfall	-2.598842

	Coefficients polynomial model
1	70.818534
Humidity3pm	-34.497306
Cloud3pm	-14.110065
Pressure3pm	-58.905587
WindSpeed3pm	-44.953044
WindDir3pm	-16.625414
Sunshine	-18.112180
Rainfall	-2.382792
Humidity3pm^2	5.297209
Humidity3pm Cloud3pm	3.811526
Humidity3pm Pressure3pm	28.136987
Humidity3pm WindSpeed3pm	12.537486
Humidity3pm WindDir3pm	-8.755339
Humidity3pm Sunshine	-7.294708
Humidity3pm Rainfall	8.673520
Cloud3pm^2	-1.263658
Cloud3pm Pressure3pm	6.184263
Cloud3pm WindSpeed3pm	11.543304
Cloud3pm WindDir3pm	1.151535
Cloud3pm Sunshine	14.325605
Cloud3pm Rainfall	11.561408
Pressure3pm^2	10.339247
Pressure3pm WindSpeed3pm	16.659560
Pressure3pm WindDir3pm	18.404142
Pressure3pm Sunshine	-1.986992
Pressure3pm Rainfall	-0.245664
WindSpeed3pm^2	5.415717
WindSpeed3pm WindDir3pm	0.445015
WindSpeed3pm Sunshine	15.305645
WindSpeed3pm Rainfall	-19.222073
WindDir3pm^2	4.630299
WindDir3pm Sunshine	0.635540
WindDir3pm Rainfall	20.616689
Sunshine^2	13.589054
Sunshine Rainfall	-5.688230
Rainfall^2	-22.396915

	Coefficients Lasso model
1	16.480539
Humidity3pm	-3.131065
Cloud3pm	2.723063
Pressure3pm	-0.000000
WindSpeed3pm	-2.157586
WindDir3pm	0.000000
Sunshine	0.000000
Rainfall	-0.000000
Humidity3pm^2	-0.000000
Humidity3pm Cloud3pm	-0.000000
Humidity3pm Pressure3pm	-0.000000
Humidity3pm WindSpeed3pm	-0.000000
Humidity3pm WindDir3pm	-0.000000
Humidity3pm Sunshine	-0.000000
Humidity3pm Rainfall	-0.000000
Cloud3pm^2	0.000000
Cloud3pm Pressure3pm	-0.000000
Cloud3pm WindSpeed3pm	-0.000000
Cloud3pm WindDir3pm	0.000000
Cloud3pm Sunshine	0.000000
Cloud3pm Rainfall	-0.000000
Pressure3pm^2	-3.868799
Pressure3pm WindSpeed3pm	-0.000000
Pressure3pm WindDir3pm	-0.000000
Pressure3pm Sunshine	-0.000000
Pressure3pm Rainfall	-0.000000
WindSpeed3pm^2	-0.000000
WindSpeed3pm WindDir3pm	-0.000000
WindSpeed3pm Sunshine	-0.000000
WindSpeed3pm Rainfall	-0.000000
WindDir3pm^2	-0.000000
WindDir3pm Sunshine	0.000000
WindDir3pm Rainfall	-0.000000
Sunshine^2	11.676768
Sunshine Rainfall	-0.000000
Rainfall^2	-0.000000

	Polynomial model	LASSO	Ridge
MAE	2.943465	4.070092	2.997895
MSE	13.838756	24.208311	13.999620

	Relative Compactness	Surface Area	Wall Area	Roof Area	Overall Height	Orientation	Galzing Area	Glazing Area Distribution	Heating Load	Cooling Load
count	768.000000	768.000000	768.000000	768.000000	768.00000	768.000000	768.000000	768.00000	768.000000	768.000000
mean	0.764167	671.708333	318.500000	176.604167	5.25000	3.500000	0.234375	2.81250	22.307201	24.587760
std	0.105777	88.086116	43.626481	45.165950	1.75114	1.118763	0.133221	1.55096	10.090196	9.513306
min	0.620000	514.500000	245.000000	110.250000	3.50000	2.000000	0.000000	0.00000	6.010000	10.900000
25%	0.682500	606.375000	294.000000	140.875000	3.50000	2.750000	0.100000	1.75000	12.992500	15.620000
50%	0.750000	673.750000	318.500000	183.750000	5.25000	3.500000	0.250000	3.00000	18.950000	22.080000
75%	0.830000	741.125000	343.000000	220.500000	7.00000	4.250000	0.400000	4.00000	31.667500	33.132500
max	0.980000	808.500000	416.500000	220.500000	7.00000	5.000000	0.400000	5.00000	43.100000	48.030000

	Relative Compactness	Surface Area	Wall Area	Roof Area	Overall Height	Orientation	Galzing Area	Glazing Area Distribution	Heating Load	Cooling Load
Relative Compactness	1.000000e+00	-9.919015e-01	-2.037817e-01	-8.688234e-01	8.277473e-01	4.678592e-17	-2.960552e-15	-7.107006e-16	0.622272	0.634339
Surface Area	-9.919015e-01	1.000000e+00	1.955016e-01	8.807195e-01	-8.581477e-01	-3.459372e-17	3.636925e-15	2.438409e-15	-0.658120	-0.672999
Wall Area	-2.037817e-01	1.955016e-01	1.000000e+00	-2.923165e-01	2.809757e-01	-2.429499e-17	-8.567455e-17	2.067384e-16	0.455671	0.427117
Roof Area	-8.688234e-01	8.807195e-01	-2.923165e-01	1.000000e+00	-9.725122e-01	-5.830058e-17	-1.759011e-15	-1.078071e-15	-0.861828	-0.862547
Overall Height	8.277473e-01	-8.581477e-01	2.809757e-01	-9.725122e-01	1.000000e+00	4.492205e-17	1.489134e-17	-2.920613e-17	0.889431	0.895785
Orientation	4.678592e-17	-3.459372e-17	-2.429499e-17	-5.830058e-17	4.492205e-17	1.000000e+00	-9.406007e-16	-2.549352e-16	-0.002587	0.014290
Galzing Area	-2.960552e-15	3.636925e-15	-8.567455e-17	-1.759011e-15	1.489134e-17	-9.406007e-16	1.000000e+00	2.129642e-01	0.269841	0.207505
Glazing Area Distribution	-7.107006e-16	2.438409e-15	2.067384e-16	-1.078071e-15	-2.920613e-17	-2.549352e-16	2.129642e-01	1.000000e+00	0.087368	0.050525
Heating Load	6.222722e-01	-6.581202e-01	4.556712e-01	-8.618283e-01	8.894307e-01	-2.586534e-03	2.698410e-01	8.736759e-02	1.000000	0.975862
Cooling Load	6.343391e-01	-6.729989e-01	4.271170e-01	-8.625466e-01	8.957852e-01	1.428960e-02	2.075050e-01	5.052512e-02	0.975862	1.000000

	Multivariate	Polynomial	Ridge	LASSO	LASSO with Cross Validation
MAE	2.228756	0.620909	1.944348	3.298777	1.973989
MSE	10.040484	0.680979	7.683001	21.672429	8.586230

	Coefficients Ridge model	Coefficients Lasso model	Coefficients polynomial model
1	21.167430	16.480539	70.818534
Humidity3pm	-3.083184	-3.131065	-34.497306
Cloud3pm	4.714199	2.723063	-14.110065
Pressure3pm	-1.346583	-0.000000	-58.905587
WindSpeed3pm	-3.411386	-2.157586	-44.953044
WindDir3pm	2.601451	0.000000	-16.625414
Sunshine	8.691005	0.000000	-18.112180
Rainfall	0.266296	-0.000000	-2.382792
Humidity3pm^2	0.421859	-0.000000	5.297209
Humidity3pm Cloud3pm	0.046893	-0.000000	3.811526
Humidity3pm Pressure3pm	-1.658249	-0.000000	28.136987
Humidity3pm WindSpeed3pm	-4.754987	-0.000000	12.537486
Humidity3pm WindDir3pm	-7.051185	-0.000000	-8.755339
Humidity3pm Sunshine	-8.350637	-0.000000	-7.294708
Humidity3pm Rainfall	1.697219	-0.000000	8.673520
Cloud3pm^2	-0.601082	0.000000	-1.263658
Cloud3pm Pressure3pm	-0.183609	-0.000000	6.184263
Cloud3pm WindSpeed3pm	-0.052566	-0.000000	11.543304
Cloud3pm WindDir3pm	-0.775338	0.000000	1.151535
Cloud3pm Sunshine	0.226056	0.000000	14.325605
Cloud3pm Rainfall	1.919206	-0.000000	11.561408
Pressure3pm^2	-5.906742	-3.868799	10.339247
Pressure3pm WindSpeed3pm	-4.299129	-0.000000	16.659560
Pressure3pm WindDir3pm	-2.454472	-0.000000	18.404142
Pressure3pm Sunshine	-7.136742	-0.000000	-1.986992
Pressure3pm Rainfall	0.211044	-0.000000	-0.245664
WindSpeed3pm^2	-1.303169	-0.000000	5.415717
WindSpeed3pm WindDir3pm	-4.157581	-0.000000	0.445015
WindSpeed3pm Sunshine	-1.053097	-0.000000	15.305645
WindSpeed3pm Rainfall	-1.131560	-0.000000	-19.222073
WindDir3pm^2	0.828722	-0.000000	4.630299
WindDir3pm Sunshine	0.343187	0.000000	0.635540
WindDir3pm Rainfall	-0.973821	-0.000000	20.616689
Sunshine^2	7.080499	11.676768	13.589054
Sunshine Rainfall	-2.841213	-0.000000	-5.688230
Rainfall^2	-2.574652	-0.000000	-22.396915

	Relative Compactness	Surface Area	Wall Area	Roof Area	Overall Height	Orientation	Heating Load	Cooling Load
0	0.98	514.5	294.0	110.25	7.0	2.0	15.55	21.33
1	0.98	514.5	294.0	110.25	7.0	3.0	15.55	21.33
2	0.98	514.5	294.0	110.25	7.0	4.0	15.55	21.33
3	0.98	514.5	294.0	110.25	7.0	5.0	15.55	21.33
4	0.90	563.5	318.5	122.50	7.0	2.0	20.84	28.28

	Coefficients multivariate model
Relative Compactness	-2.203722e+01
Surface Area	1.708328e+14
Wall Area	-9.965249e+13
Roof Area	-1.281246e+14
Overall Height	1.407812e+01
Orientation	-5.114746e-02
Galzing Area	7.898834e+00
Glazing Area Distribution	8.134766e-01

	Coefficients polynomial model
1	4.605692e+11
Relative Compactness	1.421639e+10
Surface Area	-9.310856e+10
Wall Area	6.451567e+10
Roof Area	-3.816844e+11
Overall Height	-2.240243e+11
Orientation	-7.776527e-01
Galzing Area	2.175857e+01
Glazing Area Distribution	9.577843e+00
Relative Compactness^2	-2.187030e+04
Relative Compactness Surface Area	-7.330324e+09
Relative Compactness Wall Area	4.276002e+09
Relative Compactness Roof Area	-9.940348e+09
Relative Compactness Overall Height	-1.029204e+10
Relative Compactness Orientation	1.074892e+00
Relative Compactness Galzing Area	-2.278633e-01
Relative Compactness Glazing Area Distribution	-1.636900e+00
Surface Area^2	1.748523e+10
Surface Area Wall Area	-2.222424e+10
Surface Area Roof Area	1.990837e+10
Surface Area Overall Height	2.080261e+11
Surface Area Orientation	4.122429e-01
Surface Area Galzing Area	-6.643944e+00
Surface Area Glazing Area Distribution	-2.476692e+00
Wall Area^2	7.014303e+09
Wall Area Roof Area	-2.075108e+10
Wall Area Overall Height	-1.283528e+11
Wall Area Orientation	5.625706e-01
Wall Area Galzing Area	4.015547e+00
Wall Area Glazing Area Distribution	6.690845e-01
Roof Area^2	-4.213454e+10
Roof Area Overall Height	3.274149e+11
Roof Area Orientation	2.136078e-01
Roof Area Galzing Area	-6.936222e+00
Roof Area Glazing Area Distribution	-1.945717e+00
Overall Height^2	-2.240243e+11
Overall Height Orientation	-3.270512e-01
Overall Height Galzing Area	-4.985657e+00
Overall Height Glazing Area Distribution	-1.251072e+00
Orientation^2	-3.299675e-01
Orientation Galzing Area	2.781296e-02
Orientation Glazing Area Distribution	2.947769e-01
Galzing Area^2	-3.087727e+00
Galzing Area Glazing Area Distribution	-3.680084e+00
Glazing Area Distribution^2	-3.295742e+00

	Coefficients polynomial model	Coefficients RIDGE model	Coefficients LASSO model	Coefficients LASSO-CV model
1	4.605692e+11	2.826509	10.802822	7.180690
Relative Compactness	1.421639e+10	2.557123	0.000000	0.000000
Surface Area	-9.310856e+10	2.249243	0.000000	0.000000
Wall Area	6.451567e+10	3.553134	0.151052	2.431144
Roof Area	-3.816844e+11	0.863554	0.000000	0.000000
Overall Height	-2.240243e+11	3.853848	15.739066	10.912068
Orientation	-7.776527e-01	-0.666793	0.000000	0.000000
Galzing Area	2.175857e+01	5.471534	4.349949	6.962932
Glazing Area Distribution	9.577843e+00	2.700909	0.000000	1.915740
Relative Compactness^2	-2.187030e+04	1.800994	0.000000	-0.000000
Relative Compactness Surface Area	-7.330324e+09	2.061941	0.000000	0.000000
Relative Compactness Wall Area	4.276002e+09	4.623310	0.000000	0.000000
Relative Compactness Roof Area	-9.940348e+09	-0.278403	0.000000	0.000000
Relative Compactness Overall Height	-1.029204e+10	-0.447347	0.000000	-0.000000
Relative Compactness Orientation	1.074892e+00	0.144284	0.000000	-0.000000
Relative Compactness Galzing Area	-2.278633e-01	5.811772	0.000000	0.000000
Relative Compactness Glazing Area Distribution	-1.636900e+00	2.668351	0.000000	0.000000
Surface Area^2	1.748523e+10	0.409940	0.000000	0.000000
Surface Area Wall Area	-2.222424e+10	-0.287847	0.000000	0.000000
Surface Area Roof Area	1.990837e+10	1.270299	0.000000	0.000000
Surface Area Overall Height	2.080261e+11	6.418693	0.000000	0.667460
Surface Area Orientation	4.122429e-01	0.303049	0.000000	0.000000
Surface Area Galzing Area	-6.643944e+00	1.184801	0.000000	0.000000
Surface Area Glazing Area Distribution	-2.476692e+00	1.150970	0.000000	0.000000
Wall Area^2	7.014303e+09	-3.980225	0.000000	-0.000000
Wall Area Roof Area	-2.075108e+10	3.501519	0.000000	2.878083
Wall Area Overall Height	-1.283528e+11	9.673425	0.000000	8.507564
Wall Area Orientation	5.625706e-01	0.111906	0.000000	0.000000
Wall Area Galzing Area	4.015547e+00	3.256192	0.000000	0.412138
Wall Area Glazing Area Distribution	6.690845e-01	1.565495	0.000000	0.000000
Roof Area^2	-4.213454e+10	-0.837770	0.000000	0.000000
Roof Area Overall Height	3.274149e+11	1.890893	0.000000	0.000000
Roof Area Orientation	2.136078e-01	0.168851	0.000000	0.000000
Roof Area Galzing Area	-6.936222e+00	0.263038	0.000000	-0.000000
Roof Area Glazing Area Distribution	-1.945717e+00	0.917221	0.000000	0.000000
Overall Height^2	-2.240243e+11	3.853848	0.000000	0.000000
Overall Height Orientation	-3.270512e-01	0.198285	0.000000	0.000000
Overall Height Galzing Area	-4.985657e+00	2.073087	0.000000	4.229869
Overall Height Glazing Area Distribution	-1.251072e+00	0.665912	0.000000	0.540700
Orientation^2	-3.299675e-01	-0.277550	0.000000	0.000000
Orientation Galzing Area	2.781296e-02	0.882388	0.000000	0.000000
Orientation Glazing Area Distribution	2.947769e-01	-0.013682	0.000000	0.000000
Galzing Area^2	-3.087727e+00	-1.763214	0.000000	-0.000000
Galzing Area Glazing Area Distribution	-3.680084e+00	-3.359032	0.000000	-2.680044
Glazing Area Distribution^2	-3.295742e+00	-3.119711	0.000000	-0.000000

	Degree	Sex	Salary Class
0	Apprenticeship	1	0
1	Apprenticeship	1	0
2	Master	0	1
3	Bachelor	1	0
4	Master	0	1
5	Apprenticeship	1	0
6	Bachelor	1	0
7	Bachelor	0	1
8	Master	1	1
9	Master	1	1

	Degree_Apprenticeship	Degree_Bachelor	Degree_Master
0	1.0	0.0	0.0
1	1.0	0.0	0.0
2	0.0	0.0	1.0
3	0.0	1.0	0.0
4	0.0	0.0	1.0
5	1.0	0.0	0.0
6	0.0	1.0	0.0
7	0.0	1.0	0.0
8	0.0	0.0	1.0
9	0.0	0.0	1.0

	Age	Sex	BP	Cholesterol	Na_to_K	Drug
0	23	F	HIGH	HIGH	25.355	DrugY
1	47	M	LOW	HIGH	13.093	drugC
2	47	M	LOW	HIGH	10.114	drugC
3	28	F	NORMAL	HIGH	7.798	drugX
4	61	F	LOW	HIGH	18.043	DrugY

	Age	Na_to_K
count	200.000000	200.000000
mean	44.315000	16.084485
std	16.544315	7.223956
min	15.000000	6.269000
25%	31.000000	10.445500
50%	45.000000	13.936500
75%	58.000000	19.380000
max	74.000000	38.247000

	Sex	BP	Cholesterol	Age	Na_to_K
90	1.0	2.0	0.0	62.0	16.594
163	0.0	0.0	1.0	21.0	28.632
76	0.0	0.0	0.0	36.0	11.198
113	0.0	1.0	1.0	65.0	13.769
98	1.0	0.0	1.0	20.0	35.639
...	...	...	...	...	...
69	0.0	0.0	1.0	18.0	24.276
101	0.0	0.0	0.0	45.0	12.854
74	1.0	0.0	1.0	31.0	17.069
46	0.0	0.0	0.0	37.0	13.091
19	0.0	0.0	1.0	32.0	25.974

Data Loading, Inspection and Visualization¶

Data Loading¶

Data Inspection/Visualization¶

Selecting columns¶

Selecting rows¶

Selecting both rows and columns by name (df.loc) or by position (df.iloc)¶

Boolean Indexing¶

Aggregation¶

Histograms¶

Kernel Density Estimation (KDE)¶

Analyzing the content of categorical columns¶

Pivot Tables¶

Examples¶

(Optiional) 2d histograms, density plots, and contour plots¶

Hexagonal bin plot¶

2d density and contour plots¶

Combining Plots¶

kNN and Performance Measures¶

kNN Recap¶

Distance metric ¶

Example¶

Exercise: Diabetes Classification¶

Data Scaling ¶

1. Model fitting and performance evaluation¶

2. Performance Curves¶

3. Assignment: Different Values of k¶

Regression¶

Background¶

Objective¶

Model¶

Least squares problem¶

Regularization ¶

Why LASSO leads to feature selection?¶

Solving our model: learning parameters via gradient descent¶

Implementation¶

Load and discover the dataset¶

Linear regression¶

Splitting the dataset ¶

Create and Fit model ¶

Prediction and Evaluation ¶

Multivariate linear regression¶

Splitting dataset ¶

Preprocessing: encoding categorical variables¶

Rescaling¶

Create and Fit model ¶

Prediction and evaluation ¶

Polynomial linear regression¶

Regularization ¶

Lasso¶

Ridge¶

K-fold cross validation¶

Assignment Solution¶

Model Selection KNN (SOLUTION)¶

kNN: Find the best k?¶

Prepare the data for learning¶

1. Cerate a validation set¶

2. Find optimal k using the validation set¶

3. Train a final kNN classifier¶

4. Select best k using Cross-Validation (CV)¶

5. Train a final kNN classifier based on the CV hyperparameter tuning¶

6. Hyperparameter tuning (multiple parameters)¶

Hint:¶

7. Fit the model using the selected (best) hyperparameters¶

Decision Trees and Random Forest (SOLUTION)¶

Exercise: Decision Trees¶

Load the libraries¶

Load the data¶

1. Model fitting¶

Hints:¶

Tuning tree depth with grid search CV¶

Exercise: Random Forest¶

Hints:¶

Tune the number of trees parameter using grid search¶

Importance plot¶

Hints:¶

Decision Trees and Random Forest¶

Decision Trees¶

Background¶

Intuition ¶

Decision criteria ¶

Selecting both rows and columns by name (`df.loc`) or by position (`df.iloc`)¶

Tensorflow/Keras Cheat Sheet: https://s3.amazonaws.com/assets.datacamp.com/blog_assets/Keras_Cheat_Sheet_Python.pdf ¶

	Degree_Apprenticeship	Degree_Bachelor	Degree_Master
0	1.0	0.0	0.0
1	1.0	0.0	0.0
2	0.0	0.0	1.0
3	0.0	1.0	0.0
4	0.0	0.0	1.0
5	1.0	0.0	0.0
6	0.0	1.0	0.0
7	0.0	1.0	0.0
8	0.0	0.0	1.0
9	0.0	0.0	1.0

	fixed acidity	volatile acidity	citric acid	residual sugar	chlorides	free sulfur dioxide	total sulfur dioxide	density	pH	sulphates	alcohol	quality
0	7.4	0.70	0.00	1.9	0.076	11	34	0.9978	3.51	0.56	9.4	0
1	7.8	0.88	0.00	2.6	0.098	25	67	0.9968	3.20	0.68	9.8	0
2	7.8	0.76	0.04	2.3	0.092	15	54	0.9970	3.26	0.65	9.8	0
3	11.2	0.28	0.56	1.9	0.075	17	60	0.9980	3.16	0.58	9.8	1
4	7.4	0.70	0.00	1.9	0.076	11	34	0.9978	3.51	0.56	9.4	0

	fixed acidity	volatile acidity	citric acid	residual sugar	chlorides	free sulfur dioxide	total sulfur dioxide	density	pH	sulphates	alcohol	quality
count	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000	1599.000000
mean	8.319637	0.527821	0.270976	2.538806	0.087467	15.875547	46.468418	0.996747	3.311113	0.658149	10.422983	0.534709
std	1.741096	0.179060	0.194801	1.409928	0.047065	10.460434	32.895920	0.001887	0.154386	0.169507	1.065668	0.498950
min	4.600000	0.120000	0.000000	0.900000	0.012000	1.000000	6.000000	0.990070	2.740000	0.330000	8.400000	0.000000
25%	7.100000	0.390000	0.090000	1.900000	0.070000	7.000000	22.000000	0.995600	3.210000	0.550000	9.500000	0.000000
50%	7.900000	0.520000	0.260000	2.200000	0.079000	14.000000	38.000000	0.996750	3.310000	0.620000	10.200000	1.000000
75%	9.200000	0.640000	0.420000	2.600000	0.090000	21.000000	62.000000	0.997835	3.400000	0.730000	11.100000	1.000000
max	15.900000	1.580000	1.000000	15.500000	0.611000	72.000000	289.000000	1.003690	4.010000	2.000000	14.900000	1.000000

	Coefficients logistic model
Intercept	-0.096618
fixed acidity	1.060667
volatile acidity	-3.335574
citric acid	-0.269326
residual sugar	0.596732
chlorides	-1.266971
free sulfur dioxide	0.391015
total sulfur dioxide	-2.752863
density	-0.782913
pH	-0.245513
sulphates	2.852787
alcohol	4.432689

	Probability poor-quality wine	Probability good wine
0	0.352811	0.647189
1	0.727663	0.272337
2	0.089182	0.910818
3	0.586545	0.413455

	Class 0	Class 1
Precision	0.724868	0.772512
Recall	0.740541	0.758140
F1 score	0.732620	0.765258

	X	Y	month	day	FFMC	DMC	DC	ISI	temp	RH	wind	rain
0	7	5	mar	fri	86.2	26.2	94.3	5.1	8.2	51	6.7	0.0
1	7	4	oct	tue	90.6	35.4	669.1	6.7	18.0	33	0.9	0.0
2	7	4	oct	sat	90.6	43.7	686.9	6.7	14.6	33	1.3	0.0
3	8	6	mar	fri	91.7	33.3	77.5	9.0	8.3	97	4.0	0.2
4	8	6	mar	sun	89.3	51.3	102.2	9.6	11.4	99	1.8	0.0

	X	Y	FFMC	DMC	DC	ISI	temp	RH	wind	rain	area	class
count	517.000000	517.000000	517.000000	517.000000	517.000000	517.000000	517.000000	517.000000	517.000000	517.000000	517.000000	517.000000
mean	4.669246	4.299807	90.644681	110.872340	547.940039	9.021663	18.889168	44.288201	4.017602	0.021663	12.847292	0.522244
std	2.313778	1.229900	5.520111	64.046482	248.066192	4.559477	5.806625	16.317469	1.791653	0.295959	63.655818	0.499989
min	1.000000	2.000000	18.700000	1.100000	7.900000	0.000000	2.200000	15.000000	0.400000	0.000000	0.000000	0.000000
25%	3.000000	4.000000	90.200000	68.600000	437.700000	6.500000	15.500000	33.000000	2.700000	0.000000	0.000000	0.000000
50%	4.000000	4.000000	91.600000	108.300000	664.200000	8.400000	19.300000	42.000000	4.000000	0.000000	0.520000	1.000000
75%	7.000000	5.000000	92.900000	142.400000	713.900000	10.800000	22.800000	53.000000	4.900000	0.000000	6.570000	1.000000
max	9.000000	9.000000	96.200000	291.300000	860.600000	56.100000	33.300000	100.000000	9.400000	6.400000	1090.840000	1.000000

	business_id	name	stars_y	text	categories	target
2203299	lpYFsXFrojiBZ1kbWR2lZw	Four Peaks Grill & Tap	5	Great service and food... Enjoy the atmosphere...	Food, Restaurants, American (New), Local Flavo...	True
482774	KskYqH1Bi7Z_61pH6Om8pg	Lotus of Siam	5	Lotus is one of my all time favorites in Las V...	Wine Bars, Nightlife, Restaurants, Seafood, Ca...	True
3086879	zFnPRtP7LGvr3sfxvy_dfg	Revolution Ale House	1	This place has definitely gone downhill. We fi...	Nightlife, Italian, Restaurants, Pizza, Bars	True
512115	gUR2pWQKLPgMEm_R_aI_aw	Shooters On The Water	3	Thought this might be a worn out hang out for ...	Dive Bars, American (New), American (Tradition...	True
3278073	hmYnzs8-aHbltaOOGDgmbA	Zipps Sports Grill	4	Zipps is almost always on happy hour! They hav...	Sports Bars, American (Traditional), Nightlife...	True
717572	aHACz8VHbBV5Je6q9u7g0Q	Burgers 2 Beer - Concord Twp.	1	The Concord Burgers to Beer NEEDS A NEW MANAGE...	Gastropubs, Bars, Nightlife, Restaurants, Beer...	True
590714	bXLaGCKzkQcA2hLT-JZQ1w	J. Sams Wine Bar	5	I saw the good reviews on here and wanted to c...	Restaurants, Arts & Entertainment, American (N...	True
2165889	7sPNbCx7vGAaH7SbNPZ6oA	Bachi Burger	1	So it's been about 4 weeks since I was told th...	Restaurants, Food, Asian Fusion, American (New...	True
621638	DublKfLa9Y0PguCryoDJ-Q	Cantina 1511	5	Great food & service! Stan the manager was the...	New Mexican Cuisine, Bars, Nightlife, Mexican,...	True
2048614	iH6heFdMwPXk9GIO6PwUvA	Beerhaus	5	We had fun at happy hour. Truth or dare Jenga ...	Breweries, Beer Bar, Bars, American (New), Foo...	True
12861	wKlH90YB5RYFvJ8N3pstVw	Union Standard	5	I go to Union Standard's happy hour from 4-6 a...	Seafood, Cocktail Bars, Nightlife, Restaurants...	True
1064434	B7pLK62P0rRxz25HV4RXFA	Victory Alley	4	I actually don't want to review this place and...	Bars, Burgers, Sports Bars, Chicken Wings, Nig...	True
2164784	7sPNbCx7vGAaH7SbNPZ6oA	Bachi Burger	4	Went here because of Diners, Drive-Ins and Div...	Restaurants, Food, Asian Fusion, American (New...	True
1315939	rZOzhSA5HP6IdpxuN4v66w	Nikko Japanese Restaurant & Sushi Bar	5	I absolutely love the Sushi here. The rolls ar...	Party & Event Planning, Japanese, Dance Clubs,...	True
3249081	foQZ6guS0l49trURKh7vlA	Double Barrel Roadhouse	4	The best things about this spot, are the atmos...	Cocktail Bars, Bars, Nightlife, American (Trad...	True
501133	vjlnj2qGXOQrLVLaCCF_mw	Desert Rose Pizza & Gastropub	5	Everything was wonderful, service, food, cockt...	Sports Bars, Tobacco Shops, Nightlife, Restaur...	True
3101497	90bL34o2KEes9pUnCOm7pQ	The Gladly	5	The Gladly is located on the SE corner of Came...	Bars, American (New), Venues & Event Spaces, N...	True
1862174	DYuOxkW4DtlJsTHdxdXSlg	Bahama Breeze	2	Maybe we got the wrong dishes. I got the chipo...	Laotian, Bars, Nightlife, Restaurants, Seafood...	True
1641551	jQJYvzUFsz9ytI1AzW0dyQ	Applebee's Grill + Bar	1	Walked in and no one greeting us, there was a ...	Steakhouses, Bars, Sports Bars, Nightlife, Ame...	True
3431743	pH0BLkL4cbxKzu471VZnuA	SUSHISAMBA - Las Vegas	2	A lot of experience was dampened by the servic...	Chinese, Japanese, Nightlife, Dim Sum, Asian F...	True
1972070	P7pxQFqr7yBKMMI2J51udw	Holsteins Shakes & Buns	5	I'm posting this review on behalf of my husban...	American (Traditional), Restaurants, Burgers, ...	True
1930383	VsPoQeCRYYHQrj9jbiLmtA	Russell's Pub N Grill	2	Alright folks- this place could've been better...	Restaurants, Nightlife, American (New), Bars, ...	True
1383937	l_kefVF1frmC0xRW2YkvUA	Whisky River	3	Came in here with my sister. I'm in town for a...	Nightlife, Music Venues, Cocktail Bars, Venues...	True
2471262	GZfz7YiV1fUHjsBj_8ytZA	Anthony's Sports Bar Restaurant & Catering	4	We went to this bar for the sole reason to wat...	Event Planning & Services, Nightlife, Bars, Ca...	True
1099305	2skQeu3C36VCiB653MIfrw	Bootleggers Modern American Smokehouse	2	Not what we expected and not worth the money. ...	Barbeque, Restaurants, Nightlife, Bars, Food, ...	True
1774310	xy1McNUocWlt-8DZ7Ifg9A	Pravda Vodka Bar	3	Saturday night\nI went to Pravda for my friend...	Bars, Nightlife, Lounges	True
3240924	iIok1p4qnpGAa07xoaXRQA	Field Table	4	Great service, vibe, and eats. I went for brun...	Restaurants, Nightlife, Cocktail Bars, Bars, C...	True
2576812	_T7f2wUgNlJqxsR-cR89SQ	Scaramouche Restaurant Pasta Bar & Grill	5	Hands down the best restaurant in Toronto. Ha...	Restaurants, Bars, French, Nightlife	True
2331196	8tKhimgRiNx74LbYDu9LIw	Greek Town	3	Close to a 4 star but...prawns were previously...	Wine Bars, Restaurants, Greek, Bars, Nightlife	True
898173	pHpU8lnnxMuPWRHOysuMIQ	Salut Kitchen Bar	5	In searching for an ideal venue for a group ce...	Bars, Nightlife, Beer, Wine & Spirits, Restaur...	True
...	...	...	...	...	...	...
510186	tvYID0arhN-shKGUrC1Wsg	Copacabana Brazilian Steak House	4	Meat on meat on meat!\n\nIt was my second time...	Beer, Wine & Spirits, Restaurants, Brazilian, ...	False
229587	QvltB7RjVOVRBl685azJ5g	Burntwood Tavern	1	We just left, we have been there many times bu...	American (Traditional), Restaurants	False
2011253	fcyk-PZKRqo4EUJ0vH1aNg	Capps Pizza	1	Called and ordered a pizza at 7pm and it still...	Pizza, Chicken Wings, Restaurants, Burgers	False
1039515	6ZIHxvFTHC1pvAzAS0uLDA	Lee's Sandwiches	4	Try the bbq pork sandwich... yum!! Gotta get e...	Vietnamese, Sandwiches, Food, Ice Cream & Froz...	False
2066597	spDZkD6cp0JUUm6ghIWHzA	Kitchen M	1	Would give this place 0 stars if that were pos...	Restaurants, Chinese	False
1733536	GGecutXeoEVlYKoxVo2WPA	Mixteca Mexican Food	4	It's hard to go wrong at Mixteca. At first gl...	Mexican, Restaurants	False
2740998	QDRFdG8gPPKL7r4yic8j7Q	The Original Burrito Company	4	So I've gota bad habit of ordering the same th...	Mexican, Restaurants	False
2603889	rbcfYmJtqwIkk17IeOI5Kw	BARDOT Brasserie	5	The best brunch in Las Vegas. \n\nThat sounds...	Cafes, Hostels, Brasseries, Hotels & Travel, R...	False
1023415	J2Am_nJkdicGk2S1DzwuPA	Carl's Jr	1	The worst carls jr. I have been to in the Paci...	Fast Food, Restaurants	False
1938088	kRgAf6j2y1eR0wOFdzFAuw	Firefly	4	Great bar with huge liquor selection. He tapas...	Tapas/Small Plates, Restaurants, Tapas Bars, S...	False
1231561	vWFhRvHVIJAzIeOX4g_YcA	Original Tommy's	1	All I have to say is I see why this place neve...	Restaurants, Burgers	False
3362107	BWWzh28StP6hkMm5L4nCAQ	Pita Land	2	This fast food restaurant is strangely under-s...	Middle Eastern, Sandwiches, Halal, Fast Food, ...	False
727051	j-5O-Ehd2eaCHYgmTSfoRw	Chipotle Mexican Grill	2	When I was halfway through the long line at Ch...	Mexican, Restaurants, Fast Food	False
2312132	EWpRPVSiPxWbdJBgNDNvGw	Piada Italian Street Food	5	My husband and I went to the soft opening earl...	Restaurants, Salad, Wraps, Italian	False
2859049	-_TSaVr53qiEGqMkwyEMaQ	Parsley Modern Mediterranean	4	Decent food, flavorful and filling, especially...	Sandwiches, Mediterranean, Restaurants, Middle...	False
97004	zidkKI_N1OPxsiddTOQH_Q	Naked BBQ	5	Best barbeque brisket I have ever eaten. The o...	American (New), Caterers, Southern, Restaurant...	False
852068	q3dJQtwZQrrurNT-1bNKgQ	Capo's Italian Cuisine	5	Had a great night and Armondo served us well! ...	Restaurants, Italian	False
916914	7iuruLs-q-_RW06IbsBxZw	Arirang Korean BBQ	5	I love this place. Every time I come I get the...	Korean, Barbeque, Restaurants	False
2932256	S599hCA4kJJO3_b6SRFKoA	Michoacan	1	This is the worst place I have ever been to! I...	Seafood, Restaurants, Mexican, Breakfast & Brunch	False
2475995	4JNXUYY8wbaaDmk3BPzlWw	Mon Ami Gabi	4	Visited early August 2016. \nWe came for break...	Steakhouses, Breakfast & Brunch, Restaurants, ...	False
2970425	PeATTp15Y_ExaN6mR1dmKw	Quaker Steak & Lube	4	Chicken was awesome! The Smokey Gold barbecue ...	Steakhouses, Restaurants, Chicken Wings, Ameri...	False
1176580	CPgz4srKkE5u9aoBAOaQsA	Davidson Pizza Company	4	Good selection of specialty pizzas. $18 range ...	Chicken Wings, Salad, Restaurants, Pizza	False
2786561	Jy40ercZIQaNcz2qV3qgow	Mi Amigo's Mexican Grill	5	Amazing prices, amazing food, amazing margarit...	Restaurants, Mexican	False
1147706	ysJo5Jdo29XOBCnKrbUKWg	Paris 66	4	Fine French food is rare in Pittsburgh. Fairl...	Breakfast & Brunch, Creperies, Gluten-Free, Br...	False
2061131	sGZg_8t5I5WopNwsW7riAQ	Wild Ocean Seafood Market & Grille	5	Came for the fresh, wild caught fish and fell ...	Restaurants, Seafood Markets, Food, Meat Shops...	False
2429937	Vs7gc9EE3k9wARuUcN9piA	Pan Asian	5	Pan Asian stays 5 stars for me. I went back t...	Thai, Chinese, Japanese, Restaurants	False
640067	mYzlPKXvOVRrQivHnDqD5g	YamChops	5	Amazingly delicious with a variety of distinct...	Butcher, Juice Bars & Smoothies, Vegan, Restau...	False
1566257	9Zl4uWSgSMpxHnsK_MPneg	Palermo Family Restaurant	4	Eat here quite often, the Palermo Special piz...	Restaurants, Pizza	False
1607254	F-AYOq1xIY2u_qmWUG5VBw	Hakka Ren	5	It was amazing! I had the chilli fish schezuan...	Chinese, Halal, Indian, Restaurants	False
1076163	yNBVOKSZN_AIjSJdhF_rqA	Garbanzo Mediterranean Grill	4	My salad was really good. Nice atmosphere, fri...	Falafel, Event Planning & Services, American (...	False

	bow	tfidf	l2
0	0.560429	0.701643	0.502500
1	0.711929	0.734071	0.502500
2	0.719000	0.692357	0.710357
3	0.702857	0.673286	0.727643
4	0.680357	0.665714	0.716286
5	0.667714	0.660714	0.694786